diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ab94ebf34b04..1025e4ed064f 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -15,15 +15,20 @@ concurrency: jobs: Build-and-Deploy-docs: + if: ${{ github.event.repository.name != 'scream' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: show-progress: false fetch-depth: 0 # Needed, or else gh-pages won't be fetched, and push rejected - submodules: false # speeds up clone and not building anything in submodules + # TODO: git rid of dependency on CIME + # TODO: another option to investigate is a sparse checkout. + # In the scream repo, all other components do not need to be checked out. + # And even in the upstream, we mainly need only components/xyz/docs (and a few more places). + submodules: true - name: Show action trigger - run: echo "= The job was automatically triggered by a ${{github.event_name}} event." + run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 with: @@ -31,6 +36,10 @@ jobs: - name: Install python deps run: python3 -m pip install mkdocs-material pymdown-extensions mkdocs-monorepo-plugin mdutils mkdocs-bibtex # build every time (PR or push to master) + - name: Generate EAMxx params docs + working-directory: components/eamxx/scripts + run: | + ./eamxx-params-docs-autogen - name: Build run: mkdocs build --strict --verbose # Only deploy to the main github page when there is a push to master diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml new file mode 100644 index 000000000000..1c10db1f2671 --- /dev/null +++ b/.github/workflows/eamxx-gh-pages.yml @@ -0,0 +1,88 @@ +# This workflow aims to automatically rebuild eamxx documentation +# every time the master branch is updated on github and within every PR + +name: EAMxx Docs + +on: + # Runs every time master branch is updated + push: + branches: [ master ] + # Only if docs-related files are touched + paths: + - components/eamxx/mkdocs.yml + - components/eamxx/docs/** + - components/eamxx/cime_config/namelist_defaults_scream.xml + # Runs every time a PR is open against master + pull_request: + branches: [ master ] + # Only if docs-related files are touched + paths: + - components/eamxx/mkdocs.yml + - components/eamxx/docs/** + - components/eamxx/cime_config/namelist_defaults_scream.xml + + label: + types: + - created + + workflow_dispatch: + +concurrency: + # Prevent 2+ copies of this workflow from running concurrently + group: eamxx-docs-action + +jobs: + + eamxx-docs: + if: ${{ github.event.repository.name == 'scream' }} + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + show-progress: false + # TODO: git rid of dependency on CIME + # TODO: another option to investigate is a sparse checkout. + # In the scream repo, all other components do not need to be checked out. + # And even in the upstream, we mainly need only components/xyz/docs (and a few more places). + submodules: true + + - name: Show action trigger + run: | + echo "= The job was automatically triggered by a ${{github.event_name}} event." + + - name: Set up Python 3.10 + uses: actions/setup-python@v4.7.0 + with: + python-version: "3.10" + + - name: Install Python deps + run: | + pip install mkdocs pymdown-extensions mkdocs-material mdutils + + - name: Generate EAMxx params docs + working-directory: components/eamxx/scripts + run: | + ./eamxx-params-docs-autogen + + - name: Build docs + working-directory: components/eamxx + run: | + mkdocs build --strict --verbose + + # only deploy to the main github page when there is a push to master + - if: ${{ github.event_name == 'push' }} + name: GitHub Pages action + uses: JamesIves/github-pages-deploy-action@v4 + with: + # Do not remove existing pr-preview pages + clean-exclude: pr-preview + folder: ./components/eamxx/site + + # If it's a PR from within the same repo, deploy to a preview page + - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + name: Preview docs + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: components/eamxx/site/ diff --git a/.gitignore b/.gitignore index 69f9021c5cdb..c4d2a64bc994 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,8 @@ site # Ignore emacs backup files *~ + +# Ignore mkdocs site-generated files in eamxx +components/eamxx/site/* +# Ignore auto-generated eamxx_params.md file +components/eamxx/docs/common/eamxx_params.md diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index b0f8b3139e3b..cb618cf6806e 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -14,7 +14,7 @@ TIME_ATM[%phys]_LND[%phys]_ICE[%phys]_OCN[%phys]_ROF[%phys]_GLC[%phys]_WAV[%phys][_ESP%phys][_BGC%phys] Where for the EAM specific compsets below the following is supported TIME = Time period (e.g. 2000, HIST, RCP8...) - ATM = [EAM, SATM, SCREAM] + ATM = [EAM, EAMXX, SATM, SCREAM] LND = [ELM, SLND] ICE = [MPASSI, CICE, DICE, SICE] OCN = [MPASO, DOCN, SOCN] @@ -396,6 +396,14 @@ 1850_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_MALI%STATIC_SWAV + + + + WCYCLXX2010 + 2010_EAMXX_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV + + + MPAS_LISIO_TEST diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 565b206ae912..8b0d07282696 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -1853,6 +1853,29 @@ + + + "crusher-gpu-scream ne30np4 and ne30np4.pg2" + + -2 + -2 + -2 + -2 + -2 + -2 + -2 + -2 + + + 1 + 7 + 1 + 1 + 1 + 1 + + + @@ -2336,6 +2359,30 @@ + + + 8 + 56 + + -6 + -6 + -6 + -6 + -6 + -6 + -6 + -6 + + + 1 + 7 + 1 + 1 + 1 + 1 + + + 4 diff --git a/cime_config/config_files.xml b/cime_config/config_files.xml index addb83883ca6..97d73759ac4d 100644 --- a/cime_config/config_files.xml +++ b/cime_config/config_files.xml @@ -127,6 +127,7 @@ $SRCROOT/components/stub_comps/satm $SRCROOT/components/xcpl_comps/xatm $SRCROOT/components/eam/ + $SRCROOT/components/eamxx/ $SRCROOT/components/eamxx/ case_comps diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 7cdd7f0a0557..1894ef66fa5d 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -1446,6 +1446,16 @@ oEC60to30v3 + + ne120np4.pg2 + ne120np4.pg2 + EC30to60E2r2 + null + null + null + EC30to60E2r2 + + ne120np4.pg2 r05 @@ -1486,6 +1496,16 @@ oRRS18to6v3 + + ne256np4.pg2 + ne256np4.pg2 + oRRS18to6v3 + null + null + null + oRRS18to6v3 + + ne256np4.pg2 r0125 diff --git a/cime_config/machines/Depends.frontier-gpu.crayclang.cmake b/cime_config/machines/Depends.frontier-gpu.crayclang.cmake new file mode 100644 index 000000000000..e41d959b52b4 --- /dev/null +++ b/cime_config/machines/Depends.frontier-gpu.crayclang.cmake @@ -0,0 +1,43 @@ +set(CICE_F90 + ice_FY.F90 + ice_aerosol.F90 + ice_age.F90 + ice_atmo.F90 + ice_blocks.F90 + ice_calendar.F90 + ice_diagnostics.F90 + ice_distribution.F90 + ice_domain.F90 + ice_domain_size.F90 + ice_dyn_evp.F90 + ice_fileunits.F90 + ice_flux.F90 + ice_forcing.F90 + ice_grid.F90 + ice_history.F90 + ice_history_fields.F90 + ice_init.F90 + ice_itd.F90 + ice_kinds_mod.F90 + ice_lvl.F90 + ice_mechred.F90 + ice_meltpond.F90 + ice_ocean.F90 + ice_orbital.F90 + ice_probability.F90 + ice_probability_tools.F90 + ice_read_write.F90 + ice_restoring.F90 + ice_shortwave.F90 + ice_spacecurve.F90 + ice_state.F90 + ice_step_mod.F90 + ice_therm_itd.F90 + ice_therm_vertical.F90 + ice_transport_driver.F90 + ice_transport_remap.F90 + ice_work.F90) + +foreach(ITEM IN LISTS CICE_F90) + e3sm_add_flags("cice/src/source/${ITEM}" "-O0") +endforeach() diff --git a/cime_config/machines/Depends.frontier-scream-gpu.crayclang-scream.cmake b/cime_config/machines/Depends.frontier-scream-gpu.crayclang-scream.cmake new file mode 100644 index 000000000000..be5ca2a3e2e9 --- /dev/null +++ b/cime_config/machines/Depends.frontier-scream-gpu.crayclang-scream.cmake @@ -0,0 +1,53 @@ +set(REDOPT + ../driver-mct/main/seq_io_mod.F90 + elm/src/biogeophys/BandDiagonalMod.F90) + +if (NOT DEBUG) + foreach(ITEM IN LISTS REDOPT) + e3sm_add_flags("${ITEM}" "-O1 -g") + endforeach() +endif() + +set(CICE_F90 + ice_FY.F90 + ice_aerosol.F90 + ice_age.F90 + ice_atmo.F90 + ice_blocks.F90 + ice_calendar.F90 + ice_diagnostics.F90 + ice_distribution.F90 + ice_domain.F90 + ice_domain_size.F90 + ice_dyn_evp.F90 + ice_fileunits.F90 + ice_flux.F90 + ice_forcing.F90 + ice_grid.F90 + ice_history.F90 + ice_history_fields.F90 + ice_init.F90 + ice_itd.F90 + ice_kinds_mod.F90 + ice_lvl.F90 + ice_mechred.F90 + ice_meltpond.F90 + ice_ocean.F90 + ice_orbital.F90 + ice_probability.F90 + ice_probability_tools.F90 + ice_read_write.F90 + ice_restoring.F90 + ice_shortwave.F90 + ice_spacecurve.F90 + ice_state.F90 + ice_step_mod.F90 + ice_therm_itd.F90 + ice_therm_vertical.F90 + ice_transport_driver.F90 + ice_transport_remap.F90 + ice_work.F90) + +foreach(ITEM IN LISTS CICE_F90) + e3sm_add_flags("cice/src/source/${ITEM}" "-O0") +endforeach() diff --git a/cime_config/machines/cmake_macros/crayclang-scream.cmake b/cime_config/machines/cmake_macros/crayclang-scream.cmake index ab352a7e6c58..0346104eda30 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream.cmake @@ -9,8 +9,8 @@ string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY") -# -em (default) generates MODULENAME.mod files -string(APPEND CMAKE_Fortran_FLAGS " -f free -N 255 -h byteswapio -em") +# -em -ef generates modulename.mod files (lowercase), which we must have +string(APPEND CMAKE_Fortran_FLAGS " -f free -em -ef") if (NOT compile_threaded) # -M1077 flag used to suppress message about OpenMP directives # that are ignored for non-threaded builds. (-h omp inactive) @@ -18,7 +18,8 @@ if (NOT compile_threaded) string(APPEND CMAKE_Fortran_FLAGS " -M1077") endif() set(HAS_F2008_CONTIGUOUS "TRUE") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -h byteswapio") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -ldl") +set(E3SM_LINK_WITH_FORTRAN "TRUE") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake index ef0d3303004b..2a0bfd6217c8 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake @@ -8,6 +8,5 @@ if (COMP_NAME STREQUAL elm) string(APPEND CMAKE_Fortran_FLAGS " -hfp0") endif() string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") -string(APPEND CMAKE_Fortran_FLAGS " -em -ef") set(PIO_FILESYSTEM_HINTS "gpfs") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake new file mode 100644 index 000000000000..8cc85b92cc6a --- /dev/null +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -0,0 +1,38 @@ +set(MPICC "mpicc") +set(MPICXX "mpicxx") # Needs MPICH_CXX to use hipcc +set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90 +set(SCC "cc") +set(SCXX "hipcc") +set(SFC "ftn") + +string(APPEND CPPDEFS " -DLINUX") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() + +if (compile_threaded) + string(APPEND CMAKE_C_FLAGS " -fopenmp") + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + +string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free") + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_CXX_FLAGS " -I$ENV{ROCM_PATH}/include") + +# Crusher: this resolves a crash in mct in docn init +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -hnoacc -hfp0 -hipa0") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -hnoacc -hfp0 -hipa0") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2 ") + +string(APPEND CPPDEFS " -DCPRCRAY") + +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() +set(PIO_FILESYSTEM_HINTS "lustre") + +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DCMAKE_CXX_FLAGS='-std=gnu++14'") +set(USE_HIP "TRUE") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream.cmake new file mode 100644 index 000000000000..b486efd712cc --- /dev/null +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream.cmake @@ -0,0 +1,14 @@ +if (compile_threaded) + #string(APPEND CFLAGS " -fopenmp") + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() +if (COMP_NAME STREQUAL elm) + string(APPEND CMAKE_Fortran_FLAGS " -hfp0") +endif() +string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -hsystem_alloc -f free -N 255 -h byteswapio") + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64 $ENV{OLCF_LIBUNWIND_ROOT}/lib/libunwind.a /sw/frontier/spack-envs/base/opt/cray-sles15-zen3/clang-14.0.0-rocm5.2.0/gperftools-2.10-6g5acp4pcilrl62tddbsbxlut67pp7qn/lib/libtcmalloc.a") + +set(PIO_FILESYSTEM_HINTS "gpfs") diff --git a/cime_config/machines/cmake_macros/crayclang_frontier.cmake b/cime_config/machines/cmake_macros/crayclang_frontier.cmake index 7a5fb412cbb9..6bda90a4187c 100644 --- a/cime_config/machines/cmake_macros/crayclang_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclang_frontier.cmake @@ -1,3 +1,10 @@ +if (compile_threaded) + #string(APPEND CFLAGS " -fopenmp") + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + if (COMP_NAME STREQUAL elm) # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 string(APPEND CMAKE_Fortran_FLAGS " -hfp0") diff --git a/cime_config/machines/cmake_macros/gnugpu_ascent.cmake b/cime_config/machines/cmake_macros/gnugpu_ascent.cmake index 330853a6d8a1..05805cdef74d 100644 --- a/cime_config/machines/cmake_macros/gnugpu_ascent.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_ascent.cmake @@ -1,11 +1,14 @@ string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_CUDA_FLAGS " -forward-unknown-to-host-compiler") string(APPEND CMAKE_CUDA_FLAGS_RELEASE " -O3 -arch sm_70 --use_fast_math") string(APPEND CMAKE_CUDA_FLAGS_DEBUG " -O0 -g -arch sm_70") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() +string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_VOLTA70=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ESSL_PATH}/lib64 -lessl") set(MPICXX "mpiCC") set(PIO_FILESYSTEM_HINTS "gpfs") set(USE_CUDA "TRUE") +set(CMAKE_CUDA_ARCHITECTURES "70") diff --git a/cime_config/machines/cmake_macros/gnugpu_lassen.cmake b/cime_config/machines/cmake_macros/gnugpu_lassen.cmake new file mode 100644 index 000000000000..1b87f821b73a --- /dev/null +++ b/cime_config/machines/cmake_macros/gnugpu_lassen.cmake @@ -0,0 +1,18 @@ +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_CUDA_FLAGS_RELEASE " -O3 -arch sm_70 --use_fast_math") +string(APPEND CMAKE_CUDA_FLAGS_DEBUG " -O0 -g -arch sm_70") + +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_SLASHPROC") +endif() + +string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK") + +set(PIO_FILESYSTEM_HINTS "gpfs") + +set(USE_CUDA "TRUE") + +# This may not be needed once we figure out why MPI calls are segfaulting +# on lassen when this is ON. +set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "") diff --git a/cime_config/machines/cmake_macros/gnugpu_pm-gpu.cmake b/cime_config/machines/cmake_macros/gnugpu_pm-gpu.cmake index e61e893d001b..fd52a2046502 100644 --- a/cime_config/machines/cmake_macros/gnugpu_pm-gpu.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_pm-gpu.cmake @@ -7,6 +7,7 @@ endif() string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK") string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math") string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE80=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off") +set(CMAKE_CUDA_ARCHITECTURES "80") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") set(MPICC "cc") diff --git a/cime_config/machines/cmake_macros/intel.cmake b/cime_config/machines/cmake_macros/intel.cmake index e9a808135fb2..a5cf3fea905c 100644 --- a/cime_config/machines/cmake_macros/intel.cmake +++ b/cime_config/machines/cmake_macros/intel.cmake @@ -12,6 +12,9 @@ string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -init=snan,arrays") string(APPEND CMAKE_CXX_FLAGS " -fp-model source") +if (COMP_NAME STREQUAL cice) + string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -init=nosnan,arrays") +endif() string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL") string(APPEND CMAKE_Fortran_FLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source") string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132") diff --git a/cime_config/machines/cmake_macros/intel_chrysalis.cmake b/cime_config/machines/cmake_macros/intel_chrysalis.cmake index a7e8384b01d2..a38fe5226cdc 100644 --- a/cime_config/machines/cmake_macros/intel_chrysalis.cmake +++ b/cime_config/machines/cmake_macros/intel_chrysalis.cmake @@ -18,3 +18,7 @@ if (MPILIB STREQUAL impi) set(MPICXX "mpiicpc") set(MPIFC "mpiifort") endif() +string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_ZEN2=On") +if (compile_threaded) + string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_AGGRESSIVE_VECTORIZATION=On") +endif() diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 6401d43704b2..9441a193ecec 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -371,6 +371,7 @@ --job-name={{ job_id }} --nodes=1 --ntasks={{ total_tasks }} + --cpus-per-task={{ thread_count }} --output={{ job_id }}.%j @@ -706,6 +707,13 @@ + + + pdebug + pbatch + + + /gpfs/wolf/cli115/world-shared/e3sm/tools/bsub/throttle @@ -743,6 +751,15 @@ + + /lustre/orion/cli115/world-shared/e3sm/tools/sbatch/throttle + + batch + batch + batch + + + rhel7G diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 2aaed4a761b4..faec17a7d8a9 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -244,6 +244,7 @@ cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 cmake/3.24.3 + evp-patch @@ -266,6 +267,7 @@ $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} @@ -547,6 +549,7 @@ cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 cmake/3.24.3 + evp-patch @@ -1034,11 +1037,12 @@ Linux crayclang-scream mpich - CLI133_crusher + CLI115 /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher /lustre/orion/cli115/world-shared/e3sm/inputdata /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE + /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc 8 1 @@ -1130,11 +1134,12 @@ Linux crayclang-scream mpich - CLI133_crusher + CLI115 /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher /lustre/orion/cli115/world-shared/e3sm/inputdata /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE + /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc 8 1 @@ -1207,6 +1212,92 @@ + + Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. + .*frontier.* + CNL + crayclang-scream + mpich + cli115 + /lustre/orion/proj-shared/cli115 + .* + /lustre/orion/cli115/proj-shared/$ENV{USER}/e3sm_scratch + /lustre/orion/cli115/world-shared/e3sm/inputdata + /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /lustre/orion/cli115/world-shared/e3sm/baselines/frontier/$COMPILER + /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc + 8 + 1 + slurm + e3sm + 56 + 8 + TRUE + + + srun + + -l -K -n {{ total_tasks }} -N {{ num_nodes }} + --gpus-per-node=8 --gpu-bind=closest + -c $ENV{OMP_NUM_THREADS} + + + + + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh + /usr/share/lmod/lmod/init/perl + /usr/share/lmod/lmod/init/env_modules_python.py + /usr/share/lmod/lmod/libexec/lmod perl + module + module + /usr/share/lmod/lmod/libexec/lmod python + + + PrgEnv-cray + craype-accel-amd-gfx90a + rocm/5.1.0 + libunwind/1.6.2 + + + cce/15.0.1 + craype craype/2.7.20 + cray-mpich cray-mpich/8.1.26 + cray-python/3.9.13.1 + subversion/1.14.1 + git/2.36.1 + cmake/3.21.3 + cray-hdf5-parallel/1.12.2.1 + cray-netcdf-hdf5parallel/4.9.0.1 + cray-parallel-netcdf/1.12.3.1 + darshan-runtime + + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + 0 + + $ENV{NETCDF_DIR} + $ENV{PNETCDF_DIR} + + 1 + 1 + 2 + $SHELL{which hipcc} + $ENV{CRAY_LD_LIBRARY_PATH}:$ENV{LD_LIBRARY_PATH} + True + + + + 128M + spread + threads + + + Stampede2. Intel skylake nodes at TACC. 48 cores per node, batch system is SLURM @@ -1548,7 +1639,7 @@ sems-archive-env acme-env sems-archive-git - sems-archive-cmake/3.19.1 + acme-cmake/3.26.3 acme-gcc/8.1.0 @@ -2636,6 +2727,83 @@ + + LLNL Linux Cluster, Linux, 4 V100 GPUs/node, 44 IBM P9 cpu cores/node + lassen.* + LINUX + gnugpu + spectrum-mpi + cbronze + /usr/workspace/$USER/e3sm_scratch + /usr/gdata/climdat/ccsm3data/inputdata + /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /usr/workspace/$USER/archive/$CASE + /usr/gdata/climdat/baselines/$COMPILER + 16 + lsf + donahue5 -at- llnl.gov + 40 + 40 + + + + + jsrun + + -X 1 + $SHELL{if [ {{ total_tasks }} -eq 1 ];then echo --nrs 1 --rs_per_host 1;else echo --nrs $NUM_RS --rs_per_host $RS_PER_NODE;fi} + --tasks_per_rs $SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} + -d plane:$SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} + --cpu_per_rs $ENV{CPU_PER_RS} + --gpu_per_rs $ENV{GPU_PER_RS} + --bind packed:smt:$ENV{OMP_NUM_THREADS} + --latency_priority $ENV{LTC_PRT} + --stdio_mode prepended + $ENV{JSRUN_THREAD_VARS} + $ENV{SMPIARGS} + + + + /usr/share/lmod/lmod/init/env_modules_python.py + /usr/share/lmod/lmod/init/perl + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh + module + module + /usr/share/lmod/lmod/libexec/lmod python + /usr/share/lmod/lmod/libexec/lmod perl + + + git + gcc/8.3.1 + cuda/11.8.0 + cmake/3.16.8 + spectrum-mpi + python/3.7.2 + + + /p/gpfs1/$USER/e3sm_scratch/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + + + + + -E OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -E OMP_PROC_BIND=spread -E OMP_PLACES=threads -E OMP_STACKSIZE=256M + + + y + /usr/gdata/climdat/netcdf/bin:$ENV{PATH} + /usr/gdata/climdat/netcdf/lib:$ENV{LD_LIBRARY_PATH} + /usr/gdata/climdat/netcdf + 2 + 20 + 2 + gpu-cpu + $SHELL{echo "2*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} + --smpiargs="-gpu" + + + LLNL Linux Cluster, Linux (pgi), 56 pes/node, batch system is Slurm LINUX @@ -2675,17 +2843,20 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - netcdf-fortran-parallel/4.6.0 - netcdf-c-parallel/4.9.0 + /usr/gdata/climdat/install/quartz/modulefiles + hdf5/1.12.2 + netcdf-c/4.9.0 + netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 + screamML-venv/0.0.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/tce/packages/netcdf-fortran/netcdf-fortran-4.6.0-mvapich2-2.3.7-intel-classic-2021.6.0/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0/ - + /usr/gdata/climdat/install/quartz/netcdf-fortran/ + /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + diff --git a/cime_config/tests.py b/cime_config/tests.py index 968ea47b2d94..751afa1cf002 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -562,7 +562,7 @@ "SMS_D.ne4pg2_ne4pg2.F2010-SCREAM-LR", "ERP.ne4pg2_ne4pg2.F2010-SCREAM-HR.eam-double_memleak_tol", "ERP.ne4pg2_ne4pg2.F2010-SCREAM-LR.eam-double_memleak_tol", - "ERP_R_Ln10.ne4_ne4.FDPSCREAM-ARM97", + "ERS_R_Ln10.ne4_ne4.FDPSCREAM-ARM97", ) }, @@ -579,15 +579,23 @@ "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1", "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero", "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1", - "ERS_D_Ln21.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2", + "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2", ) }, + # Tests run on exclusively on mappy for scream AT testing. These tests + # should be fast, so we limit it to low res and add some thread tests + # specifically for mappy. + "e3sm_scream_v1_at" : { + "inherit" : ("e3sm_scream_v1_lowres"), + "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1") + }, + "e3sm_scream_v1_medres" : { "time" : "02:00:00", "tests" : ( # "SMS_D_Ln2.ne30_ne30.F2000-SCREAMv1-AQP1", # Uncomment once IC file for ne30 is ready - "ERS_Ln22.ne30_ne30.F2010-SCREAMv1", + "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.scream-internal_diagnostics_level", "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1", "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels", "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash", diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index c3a8d9de607f..7dcb3ae171e5 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -87,9 +87,11 @@ if (COMP_INTERFACE STREQUAL "moab") set(CPPDEFS "${CPPDEFS} -DHAVE_MOAB") endif() -if(USE_CUDA) +if (USE_CUDA) + set(CMAKE_CUDA_COMPILER_FORCED True) enable_language(CUDA) -elseif(USE_HIP) +elseif (USE_HIP) + set(CMAKE_HIP_COMPILER_FORCED True) enable_language(HIP) endif() diff --git a/components/cmake/build_eamxx.cmake b/components/cmake/build_eamxx.cmake index 135cd640b36f..2d3b2602173c 100644 --- a/components/cmake/build_eamxx.cmake +++ b/components/cmake/build_eamxx.cmake @@ -33,6 +33,12 @@ function(build_eamxx) else() include(${SCREAM_MACH_FILE_ROOT}/${MACH}.cmake) endif() + + # The machine files may enable kokkos stuff we don't want + if (NOT compile_threaded) + set(Kokkos_ENABLE_OPENMP FALSE) + endif() + add_subdirectory("eamxx") endif() diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index 2769c94f2caa..16f1f039ebde 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -250,7 +250,9 @@ macro(build_model COMP_CLASS COMP_NAME) endforeach() # Make sure we link blas/lapack - target_link_libraries(${TARGET_NAME} BLAS::BLAS LAPACK::LAPACK) + if (NOT DEFINED ENV{SKIP_BLAS}) + target_link_libraries(${TARGET_NAME} BLAS::BLAS LAPACK::LAPACK) + endif() if (E3SM_LINK_WITH_FORTRAN) set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE Fortran) @@ -284,7 +286,7 @@ macro(build_model COMP_CLASS COMP_NAME) target_link_libraries(${TARGET_NAME} PRIVATE csm_share) if (COMP_NAME STREQUAL "eam") if (USE_YAKL) - target_link_libraries(${TARGET_NAME} PRIVATE yakl) + target_link_libraries(${TARGET_NAME} PRIVATE yakl yakl_fortran_interface) endif() if (USE_SAMXX) target_link_libraries(${TARGET_NAME} PRIVATE samxx) diff --git a/components/cmake/find_dep_packages.cmake b/components/cmake/find_dep_packages.cmake index eee4ea1426dd..9d5628bb0bd4 100644 --- a/components/cmake/find_dep_packages.cmake +++ b/components/cmake/find_dep_packages.cmake @@ -45,5 +45,9 @@ endif() find_package(PIO REQUIRED) find_package(MCT REQUIRED) find_package(CsmShare REQUIRED) -find_package(BLAS REQUIRED) -find_package(LAPACK REQUIRED) + +# Hack for unsupported blas vendors +if (NOT DEFINED ENV{SKIP_BLAS}) + find_package(BLAS REQUIRED) + find_package(LAPACK REQUIRED) +endif() diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index be37559d1e31..8dc532b6a3df 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -3814,10 +3814,8 @@ if ($shoc_sgs =~ /$TRUE/io) { add_default($nl, 'shoc_lambda_thresh'); add_default($nl, 'shoc_Ckh'); add_default($nl, 'shoc_Ckm'); - add_default($nl, 'shoc_Ckh_s_min'); - add_default($nl, 'shoc_Ckm_s_min'); - add_default($nl, 'shoc_Ckh_s_max'); - add_default($nl, 'shoc_Ckm_s_max'); + add_default($nl, 'shoc_Ckh_s'); + add_default($nl, 'shoc_Ckm_s'); } diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index 60e3199c4b22..56c816b1753c 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -829,10 +829,8 @@ 0.02D0 0.1D0 0.1D0 - 0.1D0 - 0.1D0 - 0.1D0 - 0.1D0 + 0.1D0 + 0.1D0 .false. diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index a79526cb7748..bc9977f67b58 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -3403,27 +3403,15 @@ Coefficient for eddy diffusivity of momentum. Default: set by build-namelist - -Minimum allowable value for coefficient for eddy diffusivity for heat for stable boundary layers. +Coefficient for eddy diffusivity for heat for stable boundary layers. Default: set by build-namelist - -Maximum allowable value for coefficient for eddy diffusivity for heat for stable boundary layers. -Default: set by build-namelist - - - -Minimum allowable value for coefficient for eddy diffusivity for momentum for stable boundary layers. -Default: set by build-namelist - - - -Maximum allowable value for coefficient for eddy diffusivity for momentum for stable boundary layers. +Coefficient for eddy diffusivity for momentum for stable boundary layers. Default: set by build-namelist @@ -4666,6 +4654,12 @@ Compute LS vertical transport using omega prescribed from IOP file. Default: FALSE + +Use geostropic winds specified in IOP file to apply coriolis force. +Default: FALSE + + Use relaxation for temperature and moisture. diff --git a/components/eam/src/control/iop_data_mod.F90 b/components/eam/src/control/iop_data_mod.F90 index 45504d5d7b63..f7b05dee5e09 100644 --- a/components/eam/src/control/iop_data_mod.F90 +++ b/components/eam/src/control/iop_data_mod.F90 @@ -98,8 +98,10 @@ module iop_data_mod real(r8), public :: tsair(1) ! air temperature at the surface real(r8), public :: udiff(plev) ! model minus observed uwind real(r8), public :: uobs(plev) ! actual u wind + real(r8), public :: uls(plev) ! large scale / geostropic u wind real(r8), public :: vdiff(plev) ! model minus observed vwind real(r8), public :: vobs(plev) ! actual v wind + real(r8), public :: vls(plev) ! large scale / geostropic v wind real(r8), public :: cldobs(plev) ! observed cld real(r8), public :: clwpobs(plev) ! observed clwp real(r8), public :: aldirobs(1) ! observed aldir @@ -154,6 +156,8 @@ module iop_data_mod logical*4, public :: have_tsair ! dataset contains tsair logical*4, public :: have_u ! dataset contains u logical*4, public :: have_v ! dataset contains v + logical*4, public :: have_uls ! dataset contains large scale u + logical*4, public :: have_vls ! dataset contains large scale v logical*4, public :: have_cld ! dataset contains cld logical*4, public :: have_cldliq ! dataset contains cldliq logical*4, public :: have_cldice ! dataset contains cldice @@ -166,8 +170,9 @@ module iop_data_mod logical*4, public :: have_asdif ! dataset contains asdif logical*4, public :: scm_iop_srf_prop ! use the specified surface properties logical*4, public :: iop_dosubsidence ! compute Eulerian LS vertical advection - logical*4, public :: iop_nudge_tq ! use relaxation for t and q - logical*4, public :: iop_nudge_uv ! use relaxation for u and v + logical*4, public :: iop_coriolis ! use geostropic winds to apply coriolis forcing + logical*4, public :: iop_nudge_tq! use relaxation for t and q + logical*4, public :: iop_nudge_uv! use relaxation for u and v logical*4, public :: scm_observed_aero ! use observed aerosols in SCM file logical*4, public :: precip_off ! turn off precipitation processes logical*4, public :: scm_zero_non_iop_tracers ! initialize non-IOP-specified tracers to zero @@ -184,7 +189,7 @@ module iop_data_mod subroutine iop_default_opts( scmlat_out,scmlon_out,iopfile_out, & single_column_out,scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, & iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, & - scm_observed_aero_out, iop_dosubsidence_out, & + scm_observed_aero_out, iop_dosubsidence_out, iop_coriolis_out, & scm_multcols_out, dp_crm_out, iop_perturb_high_out, & precip_off_out, scm_zero_non_iop_tracers_out) !----------------------------------------------------------------------- @@ -193,6 +198,7 @@ subroutine iop_default_opts( scmlat_out,scmlon_out,iopfile_out, & logical, intent(out), optional :: single_column_out logical, intent(out), optional :: scm_iop_srf_prop_out logical, intent(out), optional :: iop_dosubsidence_out + logical, intent(out), optional :: iop_coriolis_out logical, intent(out), optional :: iop_nudge_tq_out logical, intent(out), optional :: iop_nudge_uv_out logical, intent(out), optional :: scm_observed_aero_out @@ -211,6 +217,7 @@ subroutine iop_default_opts( scmlat_out,scmlon_out,iopfile_out, & if ( present(single_column_out) ) single_column_out = .false. if ( present(scm_iop_srf_prop_out) )scm_iop_srf_prop_out = .false. if ( present(iop_dosubsidence_out) )iop_dosubsidence_out = .false. + if ( present(iop_coriolis_out) ) iop_coriolis_out = .false. if ( present(iop_nudge_tq_out) ) iop_nudge_tq_out = .false. if ( present(iop_nudge_uv_out) ) iop_nudge_uv_out = .false. if ( present(iop_nudge_tq_low_out) ) iop_nudge_tq_low_out = 1050.0_r8 @@ -229,7 +236,7 @@ end subroutine iop_default_opts subroutine iop_setopts( scmlat_in, scmlon_in,iopfile_in,single_column_in, & scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, & iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, & - scm_observed_aero_in, iop_dosubsidence_in, & + scm_observed_aero_in, iop_dosubsidence_in, iop_coriolis_in, & scm_multcols_in, dp_crm_in, iop_perturb_high_in, & precip_off_in, scm_zero_non_iop_tracers_in) !----------------------------------------------------------------------- @@ -238,6 +245,7 @@ subroutine iop_setopts( scmlat_in, scmlon_in,iopfile_in,single_column_in, & logical, intent(in), optional :: single_column_in logical, intent(in), optional :: scm_iop_srf_prop_in logical, intent(in), optional :: iop_dosubsidence_in + logical, intent(in), optional :: iop_coriolis_in logical, intent(in), optional :: iop_nudge_tq_in logical, intent(in), optional :: iop_nudge_uv_in logical, intent(in), optional :: scm_observed_aero_in @@ -272,7 +280,11 @@ subroutine iop_setopts( scmlat_in, scmlon_in,iopfile_in,single_column_in, & if (present (iop_dosubsidence_in)) then iop_dosubsidence=iop_dosubsidence_in endif - + + if (present (iop_coriolis_in)) then + iop_coriolis=iop_coriolis_in + endif + if (present (iop_nudge_tq_in)) then iop_nudge_tq=iop_nudge_tq_in endif @@ -317,6 +329,7 @@ subroutine iop_setopts( scmlat_in, scmlon_in,iopfile_in,single_column_in, & call mpibcast(scm_iop_srf_prop,1,mpilog,0,mpicom) call mpibcast(dp_crm,1,mpilog,0,mpicom) call mpibcast(iop_dosubsidence,1,mpilog,0,mpicom) + call mpibcast(iop_coriolis,1,mpilog,0,mpicom) call mpibcast(iop_nudge_tq,1,mpilog,0,mpicom) call mpibcast(iop_nudge_uv,1,mpilog,0,mpicom) call mpibcast(iop_nudge_tq_high,1,mpir8,0,mpicom) @@ -1300,6 +1313,21 @@ subroutine readiopdata(iop_update_phase1,hyam,hybm) have_u = .true. endif + ! large scale / geostropic horizontal wind (for nudging) + call getinterpncdata( ncid, scmlat, scmlon, ioptimeidx, & + 'u_ls', have_srf, srf(1), .true. , dplevs, nlev,psobs, hyam, hybm, uls, status ) + if ( status .ne. nf90_noerr ) then + have_uls = .false. + if (iop_coriolis) then + write(iulog,*) 'Large scale / geostrophic winds required for Coriolis forcing' + write(iulog,*) 'Missing variable u_ls in the IOP file' + write(iulog,*) 'Aborting run' + call endrun + endif + else + have_uls = .true. + endif + status = nf90_inq_varid( ncid, 'vsrf', varid ) if ( status .ne. nf90_noerr ) then have_srf = .false. @@ -1318,6 +1346,21 @@ subroutine readiopdata(iop_update_phase1,hyam,hybm) endif call shr_sys_flush( iulog ) + ! large scale / geostropic meridional wind (for nudging) + call getinterpncdata( ncid, scmlat, scmlon, ioptimeidx, & + 'v_ls', have_srf, srf(1), .true. , dplevs, nlev,psobs, hyam, hybm, vls, status ) + if ( status .ne. nf90_noerr ) then + have_vls = .false. + if (iop_coriolis) then + write(iulog,*) 'Large scale / geostrophic winds required for Coriolis forcing' + write(iulog,*) 'Missing variable v_ls in the IOP file' + write(iulog,*) 'Aborting run' + call endrun + endif + else + have_vls = .true. + endif + status = nf90_inq_varid( ncid, 'Prec', varid ) if ( status .ne. nf90_noerr ) then have_prec = .false. diff --git a/components/eam/src/control/runtime_opts.F90 b/components/eam/src/control/runtime_opts.F90 index 3aa26e2ab9a1..27dd2931a097 100644 --- a/components/eam/src/control/runtime_opts.F90 +++ b/components/eam/src/control/runtime_opts.F90 @@ -172,6 +172,7 @@ module runtime_opts character(len=max_chars) iopfile logical :: scm_iop_srf_prop logical :: iop_dosubsidence +logical :: iop_coriolis logical :: iop_nudge_tq logical :: iop_nudge_uv logical :: scm_diurnal_avg @@ -337,7 +338,7 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in ! IOP namelist /cam_inparm/ iopfile, scm_iop_srf_prop, iop_nudge_tq, iop_nudge_uv, & iop_nudge_tq_low, iop_nudge_tq_high, iop_nudge_tscale, & - scm_observed_aero, precip_off, & + scm_observed_aero, precip_off, iop_coriolis, & scm_zero_non_iop_tracers, iop_perturb_high, dp_crm, & iop_dosubsidence, scm_zero_non_iop_tracers @@ -378,6 +379,8 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in call iop_default_opts(scmlat_out=scmlat,scmlon_out=scmlon, & single_column_out=single_column, & scm_iop_srf_prop_out=scm_iop_srf_prop,& + iop_dosubsidence_out=iop_dosubsidence, & + iop_coriolis_out=iop_coriolis, & iop_nudge_tq_out=iop_nudge_tq, & iop_nudge_uv_out=iop_nudge_uv, & iop_nudge_tq_low_out=iop_nudge_tq_low, & @@ -385,7 +388,6 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in iop_nudge_tscale_out=iop_nudge_tscale, & scm_observed_aero_out=scm_observed_aero, & precip_off_out=precip_off, & - iop_dosubsidence_out=iop_dosubsidence, & iop_perturb_high_out=iop_perturb_high, & scm_multcols_out=scm_multcols, & dp_crm_out=dp_crm, & @@ -463,7 +465,8 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in call iop_setopts( scmlat_in=scmlat,scmlon_in=scmlon, & iopfile_in=iopfile,single_column_in=single_column,& scm_iop_srf_prop_in=scm_iop_srf_prop,& - iop_dosubsidence_in=iop_dosubsidence,& + iop_dosubsidence_in=iop_dosubsidence,& + iop_coriolis_in=iop_coriolis,& iop_nudge_tq_in=iop_nudge_tq, & iop_nudge_uv_in=iop_nudge_uv, & iop_nudge_tq_low_in=iop_nudge_tq_low, & diff --git a/components/eam/src/dynamics/se/se_iop_intr_mod.F90 b/components/eam/src/dynamics/se/se_iop_intr_mod.F90 index 09d6acac46db..02862980053c 100644 --- a/components/eam/src/dynamics/se/se_iop_intr_mod.F90 +++ b/components/eam/src/dynamics/se/se_iop_intr_mod.F90 @@ -165,6 +165,8 @@ subroutine iop_broadcast() call mpibcast(have_q,1,mpilog,0,mpicom) call mpibcast(have_u,1,mpilog,0,mpicom) call mpibcast(have_v,1,mpilog,0,mpicom) + call mpibcast(have_uls,1,mpilog,0,mpicom) + call mpibcast(have_vls,1,mpilog,0,mpicom) call mpibcast(have_omega,1,mpilog,0,mpicom) call mpibcast(have_cldliq,1,mpilog,0,mpicom) call mpibcast(have_divt,1,mpilog,0,mpicom) @@ -182,6 +184,8 @@ subroutine iop_broadcast() call mpibcast(qobs,plev,mpir8,0,mpicom) call mpibcast(uobs,plev,mpir8,0,mpicom) call mpibcast(vobs,plev,mpir8,0,mpicom) + call mpibcast(uls,plev,mpir8,0,mpicom) + call mpibcast(vls,plev,mpir8,0,mpicom) call mpibcast(cldliqobs,plev,mpir8,0,mpicom) call mpibcast(wfld,plev,mpir8,0,mpicom) @@ -189,6 +193,7 @@ subroutine iop_broadcast() call mpibcast(divq,plev,mpir8,0,mpicom) call mpibcast(divt3d,plev,mpir8,0,mpicom) call mpibcast(divq3d,plev,mpir8,0,mpicom) + call mpibcast(scmlat,1,mpir8,0,mpicom) #endif @@ -418,6 +423,10 @@ subroutine apply_iop_forcing(elem,hvcoord,hybrid,tl,n,t_before_advance,nets,nete call outfld('QDIFF',qdiff_dyn,plon,begchunk) endif + if (iop_coriolis) then + call iop_apply_coriolis(elem,t1,nelemd_todo,np_todo,dt) + endif + call outfld('TOBS',tobs,plon,begchunk) call outfld('QOBS',qobs,plon,begchunk) call outfld('DIVQ',divq,plon,begchunk) @@ -472,6 +481,7 @@ subroutine iop_domain_relaxation(elem,hvcoord,hybrid,t1,dp,nelemd_todo,np_todo,d real (kind=real_kind), dimension(nlev) :: domain_q, domain_t, domain_u, domain_v, rtau real (kind=real_kind), dimension(nlev) :: relax_t, relax_q, relax_u, relax_v, iop_pres real (kind=real_kind), dimension(np,np,nlev) :: temperature, Rstar, pnh, exner, dp + real (kind=real_kind) :: uref, vref integer :: ie, i, j, k ! Compute pressure for IOP observations @@ -522,9 +532,18 @@ subroutine iop_domain_relaxation(elem,hvcoord,hybrid,t1,dp,nelemd_todo,np_todo,d rtau(k) = iop_nudge_tscale rtau(k) = max(dt,rtau(k)) + ! If LS/geostropic winds are available then nudge to those + if (have_uls .and. have_vls) then + uref = uls(k) + vref = vls(k) + else + uref = uobs(k) + vref = vobs(k) + endif + ! Compute relaxation for winds - relax_u(k) = -(domain_u(k) - uobs(k))/rtau(k) - relax_v(k) = -(domain_v(k) - vobs(k))/rtau(k) + relax_u(k) = -(domain_u(k) - uref)/rtau(k) + relax_v(k) = -(domain_v(k) - vref)/rtau(k) ! Restrict nudging of T and Q to certain levels if requested by user ! pmidm1 variable is in unitis of [Pa], while iop_nudge_tq_low/high @@ -584,7 +603,51 @@ subroutine iop_domain_relaxation(elem,hvcoord,hybrid,t1,dp,nelemd_todo,np_todo,d end subroutine iop_domain_relaxation -!========================================================================= +subroutine iop_apply_coriolis(elem,t1,nelemd_todo,np_todo,dt) + + ! Subroutine to provide coriolis forcing to u and v winds, using geostrophic + ! winds specified in IOP forcing file. + + use kinds, only : real_kind + use iop_data_mod + use dimensions_mod, only : np, np, nlev, npsq, nelem + use parallel_mod, only: global_shared_buf, global_shared_sum + use global_norms_mod, only: wrap_repro_sum + use hybvcoord_mod, only : hvcoord_t + use hybrid_mod, only : hybrid_t + use element_mod, only : element_t + use physical_constants, only : Cp, Rgas, DD_PI + use shr_const_mod, only: shr_const_omega + + ! Input/Output variables + type (element_t) , intent(inout), target :: elem(:) + integer, intent(in) :: nelemd_todo, np_todo, t1 + real (kind=real_kind), intent(in):: dt + + ! local variables + integer :: i,j,k, ie + + real(kind=real_kind) :: fcor, u_cor, v_cor + + ! compute coriolis force + fcor = 2._real_kind*shr_const_omega*sin(scmlat*DD_PI/180._real_kind) + + do ie=1,nelemd_todo + do j=1,np_todo + do i=1,np_todo + do k=1,nlev + + u_cor = fcor * (elem(ie)%state%v(i,j,2,k,t1) - vls(k)) + v_cor = fcor * (elem(ie)%state%v(i,j,1,k,t1) - uls(k)) + + elem(ie)%state%v(i,j,1,k,t1) = elem(ie)%state%v(i,j,1,k,t1) + u_cor * dt + elem(ie)%state%v(i,j,2,k,t1) = elem(ie)%state%v(i,j,2,k,t1) - v_cor * dt + enddo + enddo + enddo + enddo + +end subroutine iop_apply_coriolis #ifdef MODEL_THETA_L subroutine crm_resolved_turb(elem,hvcoord,hybrid,t1,& diff --git a/components/eam/src/dynamics/se/stepon.F90 b/components/eam/src/dynamics/se/stepon.F90 index 2e7630a47539..831fd6d87603 100644 --- a/components/eam/src/dynamics/se/stepon.F90 +++ b/components/eam/src/dynamics/se/stepon.F90 @@ -213,7 +213,7 @@ subroutine stepon_run1( dtime_out, phys_state, phys_tend, & ! doiopupdate set to true if model time step > next available IOP if (use_iop .and. masterproc) then - if (is_first_step()) then + if (is_first_step() .or. is_first_restart_step()) then call setiopupdate_init() else call setiopupdate diff --git a/components/eam/src/physics/cam/bfb_math.inc b/components/eam/src/physics/cam/bfb_math.inc index c95cb5bc530a..c4d49103828e 100644 --- a/components/eam/src/physics/cam/bfb_math.inc +++ b/components/eam/src/physics/cam/bfb_math.inc @@ -6,8 +6,8 @@ ! Make sure to place the following lines at the top of any modules ! that use these macros: ! -! use physics_share_f2c, only: cxx_pow, cxx_sqrt, cxx_cbrt, cxx_gamma, cxx_log, & -! cxx_log10, cxx_exp, cxx_tanh, cxx_erf +! use physics_share_f2c, only: scream_pow, scream_sqrt, scream_cbrt, scream_gamma, scream_log, & +! scream_log10, scream_exp, scream_tanh, scream_erf #ifndef SCREAM_BFB_MATH_INC #define SCREAM_BFB_MATH_INC @@ -30,16 +30,16 @@ # define bfb_tanh(val) tanh(val) # define bfb_erf(val) erf(val) #else -# define bfb_pow(base, exp) cxx_pow(base, exp) -# define bfb_sqrt(base) cxx_sqrt(base) -# define bfb_cbrt(base) cxx_cbrt(base) -# define bfb_gamma(val) cxx_gamma(val) -# define bfb_log(val) cxx_log(val) -# define bfb_log10(val) cxx_log10(val) -# define bfb_exp(val) cxx_exp(val) -# define bfb_expm1(val) cxx_expm1(val) -# define bfb_tanh(val) cxx_tanh(val) -# define bfb_erf(val) cxx_erf(val) +# define bfb_pow(base, exp) scream_pow(base, exp) +# define bfb_sqrt(base) scream_sqrt(base) +# define bfb_cbrt(base) scream_cbrt(base) +# define bfb_gamma(val) scream_gamma(val) +# define bfb_log(val) scream_log(val) +# define bfb_log10(val) scream_log10(val) +# define bfb_exp(val) scream_exp(val) +# define bfb_expm1(val) scream_expm1(val) +# define bfb_tanh(val) scream_tanh(val) +# define bfb_erf(val) scream_erf(val) #endif #endif diff --git a/components/eam/src/physics/cam/physpkg.F90 b/components/eam/src/physics/cam/physpkg.F90 index 67262ff3215e..5f942152a20f 100644 --- a/components/eam/src/physics/cam/physpkg.F90 +++ b/components/eam/src/physics/cam/physpkg.F90 @@ -2867,8 +2867,8 @@ subroutine tphysbc (ztodt, & call check_energy_chng(state, tend, "clubb_tend", nstep, ztodt, & cam_in%cflx(:,1)/cld_macmic_num_steps, flx_cnd/cld_macmic_num_steps, & det_ice/cld_macmic_num_steps, flx_heat/cld_macmic_num_steps) - - + + endif diff --git a/components/eam/src/physics/cam/shoc.F90 b/components/eam/src/physics/cam/shoc.F90 index a245516eeeeb..f354ac8c71b7 100644 --- a/components/eam/src/physics/cam/shoc.F90 +++ b/components/eam/src/physics/cam/shoc.F90 @@ -17,8 +17,8 @@ module shoc ! Bit-for-bit math functions. #ifdef SCREAM_CONFIG_IS_CMAKE - use physics_share_f2c, only: cxx_pow, cxx_sqrt, cxx_cbrt, cxx_gamma, cxx_log, & - cxx_log10, cxx_exp, cxx_erf + use physics_share_f2c, only: scream_pow, scream_sqrt, scream_cbrt, scream_gamma, scream_log, & + scream_log10, scream_exp, scream_erf #endif implicit none @@ -46,6 +46,7 @@ module shoc real(rtype) :: lice ! latent heat of fusion [J/kg] real(rtype) :: eps ! rh2o/rair - 1 [-] real(rtype) :: vk ! von karmann constant [-] +real(rtype) :: p0 ! Reference pressure, Pa !========================================================= ! Tunable parameters used in SHOC @@ -65,10 +66,8 @@ module shoc real(rtype) :: lambda_thresh = 0.02_rtype ! value to apply stability correction real(rtype) :: Ckh = 0.1_rtype ! Eddy diffusivity coefficient for heat real(rtype) :: Ckm = 0.1_rtype ! Eddy diffusivity coefficient for momentum -real(rtype) :: Ckh_s_min = 0.1_rtype ! Stable PBL diffusivity minimum for heat -real(rtype) :: Ckm_s_min = 0.1_rtype ! Stable PBL diffusivity minimum for momentum -real(rtype) :: Ckh_s_max = 0.1_rtype ! Stable PBL diffusivity maximum for heat -real(rtype) :: Ckm_s_max = 0.1_rtype ! Stable PBL diffusivity maximum for momentum +real(rtype) :: Ckh_s = 0.1_rtype ! Stable PBL diffusivity for heat +real(rtype) :: Ckm_s = 0.1_rtype ! Stable PBL diffusivity for momentum !========================================================= ! Private module parameters @@ -127,13 +126,12 @@ module shoc subroutine shoc_init( & nlev, gravit, rair, rh2o, cpair, & - zvir, latvap, latice, karman, & + zvir, latvap, latice, karman, p0_shoc, & pref_mid, nbot_shoc, ntop_shoc, & thl2tune_in, qw2tune_in, qwthl2tune_in, & w2tune_in, length_fac_in, c_diag_3rd_mom_in, & lambda_low_in, lambda_high_in, lambda_slope_in, & - lambda_thresh_in, Ckh_in, Ckm_in, Ckh_s_min_in, & - Ckm_s_min_in, Ckh_s_max_in, Ckm_s_max_in) + lambda_thresh_in, Ckh_in, Ckm_in, Ckh_s_in, Ckm_s_in) implicit none @@ -151,6 +149,7 @@ subroutine shoc_init( & real(rtype), intent(in) :: latvap ! latent heat of vaporization real(rtype), intent(in) :: latice ! latent heat of fusion real(rtype), intent(in) :: karman ! Von Karman's constant + real(rtype), intent(in) :: p0_shoc! Reference pressure, Pa real(rtype), intent(in) :: pref_mid(nlev) ! reference pressures at midpoints @@ -170,10 +169,8 @@ subroutine shoc_init( & real(rtype), intent(in), optional :: lambda_thresh_in ! value to apply stability correction real(rtype), intent(in), optional :: Ckh_in ! eddy diffusivity coefficient for heat real(rtype), intent(in), optional :: Ckm_in ! eddy diffusivity coefficient for momentum - real(rtype), intent(in), optional :: Ckh_s_min_in ! Stable PBL diffusivity minimum for heat - real(rtype), intent(in), optional :: Ckm_s_min_in ! Stable PBL diffusivity minimum for momentum - real(rtype), intent(in), optional :: Ckh_s_max_in ! Stable PBL diffusivity maximum for heat - real(rtype), intent(in), optional :: Ckm_s_max_in ! Stable PBL diffusivity maximum for momentum + real(rtype), intent(in), optional :: Ckh_s_in ! Stable PBL diffusivity for heat + real(rtype), intent(in), optional :: Ckm_s_in ! Stable PBL diffusivity for momentum integer :: k @@ -185,6 +182,7 @@ subroutine shoc_init( & lcond = latvap ! [J/kg] lice = latice ! [J/kg] vk = karman ! [-] + p0 = p0_shoc ! [Pa] ! Tunable parameters, all unitless ! override default values if value is present @@ -200,10 +198,8 @@ subroutine shoc_init( & if (present(lambda_thresh_in)) lambda_thresh=lambda_thresh_in if (present(Ckh_in)) Ckh=Ckh_in if (present(Ckm_in)) Ckm=Ckm_in - if (present(Ckh_s_min_in)) Ckh_s_min=Ckh_s_min_in - if (present(Ckm_s_min_in)) Ckm_s_min=Ckm_s_min_in - if (present(Ckh_s_max_in)) Ckh_s_max=Ckh_s_max_in - if (present(Ckm_s_max_in)) Ckm_s_max=Ckm_s_max_in + if (present(Ckh_s_in)) Ckh_s=Ckh_s_in + if (present(Ckm_s_in)) Ckm_s=Ckm_s_in ! Limit pbl height to regions below 400 mb ! npbl = max number of levels (from bottom) in pbl @@ -381,6 +377,8 @@ subroutine shoc_main ( & real(rtype) :: rho_zt(shcol,nlev) ! SHOC water vapor [kg/kg] real(rtype) :: shoc_qv(shcol,nlev) + ! SHOC temperature [K] + real(rtype) :: shoc_tabs(shcol,nlev) ! Grid difference centereted on thermo grid [m] real(rtype) :: dz_zt(shcol,nlev) @@ -462,6 +460,11 @@ subroutine shoc_main ( & shcol,nlev,qw,shoc_ql,& ! Input shoc_qv) ! Output + ! Diagnose absolute temperature + call compute_shoc_temperature(& + shcol,nlev,thetal,shoc_ql,inv_exner,& ! Input + shoc_tabs) ! Output + call shoc_diag_obklen(& shcol,uw_sfc,vw_sfc,& ! Input wthl_sfc,wqw_sfc,thetal(:shcol,nlev),& ! Input @@ -487,8 +490,8 @@ subroutine shoc_main ( & call shoc_tke(& shcol,nlev,nlevi,dtime,& ! Input wthv_sec,shoc_mix,& ! Input - dz_zi,dz_zt,pres,& ! Input - u_wind,v_wind,brunt,obklen,& ! Input + dz_zi,dz_zt,pres,shoc_tabs,& ! Input + u_wind,v_wind,brunt,& ! Input zt_grid,zi_grid,pblh,& ! Input tke,tk,tkh,& ! Input/Output isotropy) ! Output @@ -730,6 +733,59 @@ subroutine compute_shoc_vapor( & end subroutine compute_shoc_vapor +!============================================================== +! Compute temperature from SHOC prognostic/diagnostic variables + +subroutine compute_shoc_temperature( & + shcol,nlev,thetal,ql,inv_exner,& ! Input + tabs) ! Output + + ! Purpose of this subroutine is to compute temperature + ! based on SHOC's prognostic liquid water potential + ! temperature. + +#ifdef SCREAM_CONFIG_IS_CMAKE + use shoc_iso_f, only: compute_shoc_temperature_f +#endif + + implicit none + +! INPUT VARIABLES + ! number of columns [-] + integer, intent(in) :: shcol + ! number of mid-point levels [-] + integer, intent(in) :: nlev + ! liquid water potential temperature [K] + real(rtype), intent(in) :: thetal(shcol,nlev) + ! cloud water mixing ratio [kg/kg] + real(rtype), intent(in) :: ql(shcol,nlev) + ! inverse exner function [-] + real(rtype), intent(in) :: inv_exner(shcol,nlev) + +! OUTPUT VARIABLES + ! absolute temperature [K] + real(rtype), intent(out) :: tabs(shcol,nlev) + +! LOCAL VARIABLES + integer :: i, k + +#ifdef SCREAM_CONFIG_IS_CMAKE + if (use_cxx) then + call compute_shoc_temperature_f(shcol,nlev,thetal,ql,inv_exner,tabs) + return + endif +#endif + + do k = 1, nlev + do i = 1, shcol + tabs(i,k) = thetal(i,k)/inv_exner(i,k)+(lcond/cp)*ql(i,k) + enddo + enddo + + return + +end subroutine compute_shoc_temperature + !============================================================== ! Update T, q, tracers, tke, u, and v based on implicit diffusion ! Here we use a backward Euler scheme. @@ -2864,8 +2920,8 @@ subroutine shoc_assumed_pdf_compute_s(& qn=s endif endif - - ! Prevent possibility of empty clouds or rare occurence of + + ! Prevent possibility of empty clouds or rare occurence of ! cloud liquid less than zero if (qn .le. 0._rtype) then C=0._rtype @@ -2969,8 +3025,8 @@ end subroutine shoc_assumed_pdf_compute_buoyancy_flux subroutine shoc_tke(& shcol,nlev,nlevi,dtime,& ! Input wthv_sec,shoc_mix,& ! Input - dz_zi,dz_zt,pres,& ! Input - u_wind,v_wind,brunt,obklen,&! Input + dz_zi,dz_zt,pres,tabs,& ! Input + u_wind,v_wind,brunt,& ! Input zt_grid,zi_grid,pblh,& ! Input tke,tk,tkh, & ! Input/Output isotropy) ! Output @@ -2997,14 +3053,14 @@ subroutine shoc_tke(& real(rtype), intent(in) :: u_wind(shcol,nlev) ! Zonal wind [m/s] real(rtype), intent(in) :: v_wind(shcol,nlev) - ! Obukov length - real(rtype), intent(in) :: obklen(shcol) ! thickness on interface grid [m] real(rtype), intent(in) :: dz_zi(shcol,nlevi) ! thickness on thermodynamic grid [m] real(rtype), intent(in) :: dz_zt(shcol,nlev) ! pressure [Pa] real(rtype), intent(in) :: pres(shcol,nlev) + ! absolute temperature [K] + real(rtype), intent(in) :: tabs(shcol,nlev) ! Brunt Vaisalla frequncy [/s] real(rtype), intent(in) :: brunt(shcol,nlev) ! heights on midpoint grid [m] @@ -3052,7 +3108,7 @@ subroutine shoc_tke(& call isotropic_ts(nlev, shcol, brunt_int, tke, a_diss, brunt, isotropy) !Compute eddy diffusivity for heat and momentum - call eddy_diffusivities(nlev, shcol, obklen, pblh, zt_grid, & + call eddy_diffusivities(nlev, shcol, pblh, zt_grid, tabs, & shoc_mix, sterm_zt, isotropy, tke, tkh, tk) return @@ -3316,7 +3372,7 @@ subroutine isotropic_ts(nlev, shcol, brunt_int, tke, a_diss, brunt, isotropy) end subroutine isotropic_ts -subroutine eddy_diffusivities(nlev, shcol, obklen, pblh, zt_grid, & +subroutine eddy_diffusivities(nlev, shcol, pblh, zt_grid, tabs, & shoc_mix, sterm_zt, isotropy, tke, tkh, tk) !------------------------------------------------------------ @@ -3332,12 +3388,12 @@ subroutine eddy_diffusivities(nlev, shcol, obklen, pblh, zt_grid, & !intent-ins integer, intent(in) :: nlev, shcol - ! Monin-Okbukov length [m] - real(rtype), intent(in) :: obklen(shcol) ! PBL height [m] real(rtype), intent(in) :: pblh(shcol) ! Heights on the mid-point grid [m] real(rtype), intent(in) :: zt_grid(shcol,nlev) + ! Absolute temperature [K] + real(rtype), intent(in) :: tabs(shcol,nlev) ! Mixing length [m] real(rtype), intent(in) :: shoc_mix(shcol,nlev) ! Interpolate shear production to thermo grid @@ -3355,48 +3411,30 @@ subroutine eddy_diffusivities(nlev, shcol, obklen, pblh, zt_grid, & !local vars integer :: i, k - real(rtype) :: z_over_L, zt_grid_1d(shcol) - real(rtype) :: Ckh_s, Ckm_s !parameters - ! Critical value of dimensionless Monin-Obukhov length, - ! for which diffusivities are no longer damped - real(rtype), parameter :: zL_crit_val = 100.0_rtype + ! Minimum absolute temperature threshold for which to apply extra mixing [K] + real(rtype), parameter :: temp_crit = 182.0_rtype ! Transition depth [m] above PBL top to allow ! stability diffusivities real(rtype), parameter :: pbl_trans = 200.0_rtype #ifdef SCREAM_CONFIG_IS_CMAKE if (use_cxx) then - call eddy_diffusivities_f(nlev, shcol, obklen, pblh, zt_grid, & + call eddy_diffusivities_f(nlev, shcol, pblh, zt_grid, tabs, & shoc_mix, sterm_zt, isotropy, tke, tkh, tk) return endif #endif - !store zt_grid at nlev in 1d array - zt_grid_1d(1:shcol) = zt_grid(1:shcol,nlev) - do k = 1, nlev do i = 1, shcol - ! Dimensionless Okukhov length considering only - ! the lowest model grid layer height to scale - z_over_L = zt_grid_1d(i)/obklen(i) + if (tabs(i,nlev) .lt. temp_crit .and. (zt_grid(i,k) .lt. pblh(i)+pbl_trans)) then + ! If surface layer temperature is running away, apply extra mixing + ! based on traditional stable PBL diffusivities that are not damped + ! by stability functions. - if (z_over_L .gt. 0._rtype .and. (zt_grid(i,k) .lt. pblh(i)+pbl_trans)) then - ! If surface layer is stable, based on near surface - ! dimensionless Monin-Obukov use modified coefficients of - ! tkh and tk that are primarily based on shear production - ! and SHOC length scale, to promote mixing within the PBL - ! and to a height slighty above to ensure smooth transition. - - ! Compute diffusivity coefficient as function of dimensionless - ! Obukhov, given a critical value - Ckh_s = max(Ckh_s_min,min(Ckh_s_max,z_over_L/zL_crit_val)) - Ckm_s = max(Ckm_s_min,min(Ckm_s_max,z_over_L/zL_crit_val)) - - ! Compute stable PBL diffusivities tkh(i,k) = Ckh_s*bfb_square(shoc_mix(i,k))*bfb_sqrt(sterm_zt(i,k)) tk(i,k) = Ckm_s*bfb_square(shoc_mix(i,k))*bfb_sqrt(sterm_zt(i,k)) else @@ -3740,6 +3778,9 @@ subroutine shoc_energy_integrals(& do k=1,nlev do i=1,shcol rvm = rtm(i,k) - rcm(i,k) ! compute water vapor + +!technically wrong, need to remove gz from geopotential +!but shoc does not change gz term se_int(i) = se_int(i) + host_dse(i,k)*pdel(i,k)/ggr ke_int(i) = ke_int(i) + 0.5_rtype*(bfb_square(u_wind(i,k))+bfb_square(v_wind(i,k)))*pdel(i,k)/ggr wv_int(i) = wv_int(i) + rvm*pdel(i,k)/ggr @@ -3896,7 +3937,7 @@ subroutine shoc_energy_fixer(& zt_grid,zi_grid,& ! Input se_b,ke_b,wv_b,wl_b,& ! Input se_a,ke_a,wv_a,wl_a,& ! Input - wthl_sfc,wqw_sfc,rho_zt,& ! Input + wthl_sfc,wqw_sfc,rho_zt,pint,& ! Input te_a, te_b) ! Output call shoc_energy_threshold_fixer(& @@ -3921,7 +3962,7 @@ subroutine shoc_energy_total_fixer(& zt_grid,zi_grid,& ! Input se_b,ke_b,wv_b,wl_b,& ! Input se_a,ke_a,wv_a,wl_a,& ! Input - wthl_sfc,wqw_sfc,rho_zt,& ! Input + wthl_sfc,wqw_sfc,rho_zt,pint,& ! Input te_a, te_b) ! Output implicit none @@ -3963,6 +4004,8 @@ subroutine shoc_energy_total_fixer(& real(rtype), intent(in) :: zi_grid(shcol,nlevi) ! density on midpoint grid [kg/m^3] real(rtype), intent(in) :: rho_zt(shcol,nlev) + ! pressure on interface grid [Pa] + real(rtype), intent(in) :: pint(shcol,nlevi) ! OUTPUT VARIABLES real(rtype), intent(out) :: te_a(shcol) @@ -3972,7 +4015,7 @@ subroutine shoc_energy_total_fixer(& ! density on interface grid [kg/m^3] real(rtype) :: rho_zi(shcol,nlevi) ! sensible and latent heat fluxes [W/m^2] - real(rtype) :: shf, lhf, hdtime + real(rtype) :: shf, lhf, hdtime, exner_surf integer :: i ! compute the host timestep @@ -3983,8 +4026,10 @@ subroutine shoc_energy_total_fixer(& ! Based on these integrals, compute the total energy before and after SHOC ! call do i=1,shcol - ! convert shf and lhf to W/m^2 - shf=wthl_sfc(i)*cp*rho_zi(i,nlevi) + ! convert shf and lhf + exner_surf = bfb_pow(pint(i,nlevi)/p0, rgas/cp) + shf=wthl_sfc(i)*cp*rho_zi(i,nlevi)*exner_surf + lhf=wqw_sfc(i)*rho_zi(i,nlevi) te_a(i) = se_a(i) + ke_a(i) + (lcond+lice)*wv_a(i)+lice*wl_a(i) te_b(i) = se_b(i) + ke_b(i) + (lcond+lice)*wv_b(i)+lice*wl_b(i) diff --git a/components/eam/src/physics/cam/shoc_intr.F90 b/components/eam/src/physics/cam/shoc_intr.F90 index a1d4db53181d..f008ffec3e0d 100644 --- a/components/eam/src/physics/cam/shoc_intr.F90 +++ b/components/eam/src/physics/cam/shoc_intr.F90 @@ -6,12 +6,12 @@ module shoc_intr ! by Peter Bogenschutz (Bogenschutz and Krueger 2013). ! ! ! ! SHOC replaces the exisiting turbulence, shallow convection, and ! - ! macrophysics in E3SM ! - ! ! + ! macrophysics in E3SM ! + ! ! ! ! !---------------------------Code history---------------------------- ! - ! Authors: P. Bogenschutz ! - ! ! + ! Authors: P. Bogenschutz ! + ! ! !------------------------------------------------------------------- ! use shr_kind_mod, only: r8=>shr_kind_r8 @@ -19,18 +19,18 @@ module shoc_intr use ppgrid, only: pver, pverp use phys_control, only: phys_getopts use physconst, only: rair, cpair, gravit, latvap, latice, zvir, & - rh2o, karman, tms_orocnst, tms_z0fac + rh2o, karman, tms_orocnst, tms_z0fac use constituents, only: pcnst, cnst_add, stateq_names=>cnst_name use pbl_utils, only: calc_ustar, calc_obklen use perf_mod, only: t_startf, t_stopf - use cam_logfile, only: iulog - use shoc, only: linear_interp, largeneg + use cam_logfile, only: iulog + use shoc, only: linear_interp, largeneg use spmd_utils, only: masterproc use cam_abortutils, only: endrun - - implicit none - public :: shoc_init_cnst, shoc_implements_cnst + implicit none + + public :: shoc_init_cnst, shoc_implements_cnst ! define physics buffer indicies here integer :: tke_idx, & ! turbulent kinetic energy @@ -38,7 +38,7 @@ module shoc_intr tk_idx, & wthv_idx, & ! buoyancy flux cld_idx, & ! Cloud fraction - tot_cloud_frac_idx, & ! Cloud fraction with higher ice threshold + tot_cloud_frac_idx, & ! Cloud fraction with higher ice threshold concld_idx, & ! Convective cloud fraction ast_idx, & ! Stratiform cloud fraction alst_idx, & ! Liquid stratiform cloud fraction @@ -62,11 +62,11 @@ module shoc_intr fice_idx, & vmag_gust_idx, & ixq ! water vapor index in state%q array - + integer :: ixtke ! SHOC_TKE index in state%q array integer :: cmfmc_sh_idx = 0 - + real(r8), parameter :: tke_tol = 0.0004_r8 real(r8), parameter :: & @@ -78,31 +78,22 @@ module shoc_intr shoc_liq_deep = 8.e-6, & shoc_liq_sh = 10.e-6, & shoc_ice_deep = 25.e-6, & - shoc_ice_sh = 50.e-6 - + shoc_ice_sh = 50.e-6 + logical :: lq(pcnst) - !lq_dry_wet_cnvr is true for all the water based scalars used by SHOC. - !These water based scalars will participate in dry/wet mmr conversion - logical :: lq_dry_wet_cnvr(pcnst) - logical :: history_budget - integer :: history_budget_histfile_num + integer :: history_budget_histfile_num logical :: micro_do_icesupersat - !Store names of the state%q array scalars (as they appear in the state%q array) - !which should be "excluded" from wet<->dry mmr conversion - !NOTE: Scalar name should be exactly same as it appear in the state%q array - character(len=8), parameter :: dry_wet_exclude_scalars(1) = ['SHOC_TKE'] - character(len=16) :: eddy_scheme ! Default set in phys_control.F90 - character(len=16) :: deep_scheme ! Default set in phys_control.F90 - + character(len=16) :: deep_scheme ! Default set in phys_control.F90 + real(r8), parameter :: unset_r8 = huge(1.0_r8) - + real(r8) :: shoc_timestep = unset_r8 ! Default SHOC timestep set in namelist real(r8) :: dp1 - + real(r8) :: shoc_thl2tune = unset_r8 real(r8) :: shoc_qw2tune = unset_r8 real(r8) :: shoc_qwthl2tune = unset_r8 @@ -115,52 +106,50 @@ module shoc_intr real(r8) :: shoc_lambda_thresh = unset_r8 real(r8) :: shoc_Ckh = unset_r8 real(r8) :: shoc_Ckm = unset_r8 - real(r8) :: shoc_Ckh_s_min = unset_r8 - real(r8) :: shoc_Ckm_s_min = unset_r8 - real(r8) :: shoc_Ckh_s_max = unset_r8 - real(r8) :: shoc_Ckm_s_max = unset_r8 + real(r8) :: shoc_Ckh_s = unset_r8 + real(r8) :: shoc_Ckm_s = unset_r8 integer :: edsclr_dim - + logical :: prog_modal_aero real(r8) :: micro_mg_accre_enhan_fac = huge(1.0_r8) !Accretion enhancement factor from namelist - + integer, parameter :: ncnst=1 character(len=8) :: cnst_names(ncnst) logical :: do_cnst=.true. - + logical :: liqcf_fix = .FALSE. ! HW for liquid cloud fraction fix - logical :: relvar_fix = .FALSE. !PMA for relvar fix - + logical :: relvar_fix = .FALSE. !PMA for relvar fix + contains - + ! =============================================================================== ! ! ! ! =============================================================================== ! - + subroutine shoc_register_e3sm() #ifdef SHOC_SGS ! Add SHOC fields to pbuf use physics_buffer, only: pbuf_add_field, dtype_r8, dyn_time_lvls - use ppgrid, only: pver, pverp, pcols - + use ppgrid, only: pver, pverp, pcols + call phys_getopts( eddy_scheme_out = eddy_scheme, & - deep_scheme_out = deep_scheme, & + deep_scheme_out = deep_scheme, & history_budget_out = history_budget, & history_budget_histfile_num_out = history_budget_histfile_num, & micro_do_icesupersat_out = micro_do_icesupersat, & - micro_mg_accre_enhan_fac_out = micro_mg_accre_enhan_fac) - - cnst_names=(/'TKE '/) - + micro_mg_accre_enhan_fac_out = micro_mg_accre_enhan_fac) + + cnst_names=(/'TKE '/) + ! TKE is prognostic in SHOC and should be advected by dynamics call cnst_add('SHOC_TKE',0._r8,0._r8,0._r8,ixtke,longname='turbulent kinetic energy',cam_outfld=.false.) - + ! Fields that are not prognostic should be added to PBUF call pbuf_add_field('WTHV', 'global', dtype_r8, (/pcols,pver,dyn_time_lvls/), wthv_idx) - call pbuf_add_field('TKH', 'global', dtype_r8, (/pcols,pver,dyn_time_lvls/), tkh_idx) - call pbuf_add_field('TK', 'global', dtype_r8, (/pcols,pver,dyn_time_lvls/), tk_idx) + call pbuf_add_field('TKH', 'global', dtype_r8, (/pcols,pver,dyn_time_lvls/), tkh_idx) + call pbuf_add_field('TK', 'global', dtype_r8, (/pcols,pver,dyn_time_lvls/), tk_idx) call pbuf_add_field('pblh', 'global', dtype_r8, (/pcols/), pblh_idx) call pbuf_add_field('tke', 'global', dtype_r8, (/pcols, pverp/), tke_idx) @@ -178,70 +167,70 @@ subroutine shoc_register_e3sm() call pbuf_add_field('FICE', 'physpkg',dtype_r8, (/pcols,pver/), fice_idx) call pbuf_add_field('RAD_CLUBB', 'global', dtype_r8, (/pcols,pver/), radf_idx) call pbuf_add_field('CMELIQ', 'physpkg',dtype_r8, (/pcols,pver/), cmeliq_idx) - + call pbuf_add_field('vmag_gust', 'global', dtype_r8, (/pcols/), vmag_gust_idx) - + #endif - + end subroutine shoc_register_e3sm ! =============================================================================== ! ! ! ! =============================================================================== ! - + function shoc_implements_cnst(name) !-------------------------------------------------------------------- ! Return true if specified constituent is implemented by this package !-------------------------------------------------------------------- - character(len=*), intent(in) :: name + character(len=*), intent(in) :: name logical :: shoc_implements_cnst shoc_implements_cnst = (do_cnst .and. any(name == cnst_names)) end function shoc_implements_cnst - + subroutine shoc_init_cnst(name, q, gcid) - + !------------------------------------------------------------------- ! ! Initialize the state for SHOC's prognostic variable ! !------------------------------------------------------------------- ! - + character(len=*), intent(in) :: name ! constituent name real(r8), intent(out) :: q(:,:) ! mass mixing ratio (gcol, plev) integer, intent(in) :: gcid(:) ! global column id - + #ifdef SHOC_SGS if (trim(name) == trim('SHOC_TKE')) q = tke_tol -#endif +#endif end subroutine shoc_init_cnst - + ! =============================================================================== ! ! ! ! =============================================================================== ! - + subroutine shoc_readnl(nlfile) - + !------------------------------------------------------------------- ! ! Read in any namelist parameters here ! ! (currently none) ! - !------------------------------------------------------------------- ! + !------------------------------------------------------------------- ! use units, only: getunit, freeunit use namelist_utils, only: find_group_name use mpishorthand character(len=*), intent(in) :: nlfile ! filepath for file containing namelist input - + integer :: iunit, read_status - + namelist /shocpbl_diff_nl/ shoc_timestep, shoc_thl2tune, shoc_qw2tune, shoc_qwthl2tune, & shoc_w2tune, shoc_length_fac, shoc_c_diag_3rd_mom, & shoc_lambda_low, shoc_lambda_high, shoc_lambda_slope, & - shoc_lambda_thresh, shoc_Ckh, shoc_Ckm, shoc_Ckh_s_min, & - shoc_Ckm_s_min, shoc_Ckh_s_max, shoc_Ckm_s_max - + shoc_lambda_thresh, shoc_Ckh, shoc_Ckm, shoc_Ckh_s, & + shoc_Ckm_s + ! Read namelist to determine if SHOC history should be called if (masterproc) then iunit = getunit() @@ -257,8 +246,8 @@ subroutine shoc_readnl(nlfile) close(unit=iunit) call freeunit(iunit) - end if - + end if + #ifdef SPMD ! Broadcast namelist variables call mpibcast(shoc_timestep, 1, mpir8, 0, mpicom) @@ -274,23 +263,21 @@ subroutine shoc_readnl(nlfile) call mpibcast(shoc_lambda_thresh, 1, mpir8, 0, mpicom) call mpibcast(shoc_Ckh, 1, mpir8, 0, mpicom) call mpibcast(shoc_Ckm, 1, mpir8, 0, mpicom) - call mpibcast(shoc_Ckh_s_min, 1, mpir8, 0, mpicom) - call mpibcast(shoc_Ckm_s_min, 1, mpir8, 0, mpicom) - call mpibcast(shoc_Ckh_s_max, 1, mpir8, 0, mpicom) - call mpibcast(shoc_Ckm_s_max, 1, mpir8, 0, mpicom) + call mpibcast(shoc_Ckh_s, 1, mpir8, 0, mpicom) + call mpibcast(shoc_Ckm_s, 1, mpir8, 0, mpicom) #endif - + end subroutine shoc_readnl - + ! =============================================================================== ! ! ! ! =============================================================================== ! - + subroutine shoc_init_e3sm(pbuf2d, dp1_in) !------------------------------------------------------------------- ! ! Initialize SHOC for E3SM ! - !------------------------------------------------------------------- ! + !------------------------------------------------------------------- ! use physics_types, only: physics_state, physics_ptend use ppgrid, only: pver, pverp, pcols @@ -300,46 +287,45 @@ subroutine shoc_init_e3sm(pbuf2d, dp1_in) use physics_buffer, only: pbuf_get_index, pbuf_set_field, & physics_buffer_desc use rad_constituents, only: rad_cnst_get_info, rad_cnst_get_mode_num_idx, & - rad_cnst_get_mam_mmr_idx - use constituents, only: cnst_get_ind - use shoc, only: shoc_init + rad_cnst_get_mam_mmr_idx + use constituents, only: cnst_get_ind + use shoc, only: shoc_init use cam_history, only: horiz_only, addfld, add_default use error_messages, only: handle_errmsg - use trb_mtn_stress, only: init_tms - + use trb_mtn_stress, only: init_tms + implicit none ! Input Variables type(physics_buffer_desc), pointer :: pbuf2d(:,:) - + real(r8) :: dp1_in - + integer :: lptr integer :: nmodes, nspec, m, l, icnst, idw integer :: ixnumliq integer :: ntop_shoc integer :: nbot_shoc - integer :: sz_dw_sclr !size of dry<->wet conversion excluded scalar array - character(len=128) :: errstring + character(len=128) :: errstring logical :: history_amwg - + lq(1:pcnst) = .true. edsclr_dim = pcnst - + !----- Begin Code ----- call cnst_get_ind('Q',ixq) ! get water vapor index from the state%q array ! ----------------------------------------------------------------- ! - ! Determine how many constituents SHOC will transport. Note that - ! SHOC does not transport aerosol consituents. Therefore, need to + ! Determine how many constituents SHOC will transport. Note that + ! SHOC does not transport aerosol consituents. Therefore, need to ! determine how many aerosols constituents there are and subtract that - ! off of pcnst (the total consituents) + ! off of pcnst (the total consituents) ! ----------------------------------------------------------------- ! call phys_getopts(prog_modal_aero_out=prog_modal_aero, & history_amwg_out = history_amwg, & - liqcf_fix_out = liqcf_fix) - + liqcf_fix_out = liqcf_fix) + ! Define physics buffers indexes cld_idx = pbuf_get_index('CLD') ! Cloud fraction tot_cloud_frac_idx = pbuf_get_index('TOT_CLOUD_FRAC') ! Cloud fraction @@ -347,7 +333,7 @@ subroutine shoc_init_e3sm(pbuf2d, dp1_in) ast_idx = pbuf_get_index('AST') ! Stratiform cloud fraction alst_idx = pbuf_get_index('ALST') ! Liquid stratiform cloud fraction aist_idx = pbuf_get_index('AIST') ! Ice stratiform cloud fraction - qlst_idx = pbuf_get_index('QLST') ! Physical in-stratus LWC + qlst_idx = pbuf_get_index('QLST') ! Physical in-stratus LWC qist_idx = pbuf_get_index('QIST') ! Physical in-stratus IWC dp_frac_idx = pbuf_get_index('DP_FRAC') ! Deep convection cloud fraction icwmrdp_idx = pbuf_get_index('ICWMRDP') ! In-cloud deep convective mixing ratio @@ -357,31 +343,31 @@ subroutine shoc_init_e3sm(pbuf2d, dp1_in) prer_evap_idx = pbuf_get_index('PRER_EVAP') qrl_idx = pbuf_get_index('QRL') cmfmc_sh_idx = pbuf_get_index('CMFMC_SH') - tke_idx = pbuf_get_index('tke') + tke_idx = pbuf_get_index('tke') vmag_gust_idx = pbuf_get_index('vmag_gust') - + if (is_first_step()) then - call pbuf_set_field(pbuf2d, wthv_idx, 0.0_r8) - call pbuf_set_field(pbuf2d, tkh_idx, 0.0_r8) - call pbuf_set_field(pbuf2d, tk_idx, 0.0_r8) + call pbuf_set_field(pbuf2d, wthv_idx, 0.0_r8) + call pbuf_set_field(pbuf2d, tkh_idx, 0.0_r8) + call pbuf_set_field(pbuf2d, tk_idx, 0.0_r8) call pbuf_set_field(pbuf2d, fice_idx, 0.0_r8) call pbuf_set_field(pbuf2d, tke_idx, tke_tol) call pbuf_set_field(pbuf2d, alst_idx, 0.0_r8) call pbuf_set_field(pbuf2d, aist_idx, 0.0_r8) - + call pbuf_set_field(pbuf2d, vmag_gust_idx, 1.0_r8) - + endif - + if (prog_modal_aero) then ! Turn off modal aerosols and decrement edsclr_dim accordingly call rad_cnst_get_info(0, nmodes=nmodes) - + do m = 1, nmodes call rad_cnst_get_mode_num_idx(m, lptr) lq(lptr)=.false. edsclr_dim = edsclr_dim-1 - + call rad_cnst_get_info(0, m, nspec=nspec) do l = 1, nspec call rad_cnst_get_mam_mmr_idx(m, l, lptr) @@ -389,31 +375,14 @@ subroutine shoc_init_e3sm(pbuf2d, dp1_in) edsclr_dim = edsclr_dim-1 end do end do - + ! In addition, if running with MAM, droplet number is transported ! in dropmixnuc, therefore we do NOT want SHOC to apply transport ! tendencies to avoid double counted. Else, we apply tendencies. call cnst_get_ind('NUMLIQ',ixnumliq) lq(ixnumliq) = .false. edsclr_dim = edsclr_dim-1 - endif - - !SHOC needs all its water based scalars in terms of "dry" mmr - !but the state vector has all its scalars in terms of "wet" mmr - !By default, we will include all scalars for dry<->wet conversion - !Identify scalars which should be "excluded" from the dry<->wet conversions - - lq_dry_wet_cnvr(:) = .true. ! lets assume .true. (i.e., all scalars will participate in the conversion process) by default - sz_dw_sclr = size(dry_wet_exclude_scalars) !size of dry-wet excluded scalar array - do idw = 1, sz_dw_sclr - do icnst = 1, pcnst - if(trim(adjustl(stateq_names(icnst))) == trim(adjustl(dry_wet_exclude_scalars(idw))) )then - !This "icnst" scalar will NOT participate in dry<->wet conversion - lq_dry_wet_cnvr(icnst) = .false. - exit ! exit the loop if we found it! - endif - enddo - enddo + endif ! Add SHOC fields call addfld('SHOC_TKE', (/'lev'/), 'A', 'm2/s2', 'TKE') @@ -441,6 +410,7 @@ subroutine shoc_init_e3sm(pbuf2d, dp1_in) call addfld('PRECIPITATING_ICE_FRAC',(/'lev'/), 'A', 'fraction', 'Precipitating ice fraction') call addfld('LIQ_CLOUD_FRAC',(/'lev'/), 'A', 'fraction', 'Liquid cloud fraction') call addfld('TOT_CLOUD_FRAC',(/'lev'/), 'A', 'fraction', 'total cloud fraction') + call addfld('PBLH',horiz_only,'A','m','PBL height') call add_default('SHOC_TKE', 1, ' ') call add_default('WTHV_SEC', 1, ' ') @@ -471,69 +441,68 @@ subroutine shoc_init_e3sm(pbuf2d, dp1_in) ! ---------------------------------------------------------------! ! Initialize SHOC ! ! ---------------------------------------------------------------! - + ntop_shoc = 1 ! if >1, must be <= nbot_molec - nbot_shoc = pver ! currently always pver - + nbot_shoc = pver ! currently always pver + call shoc_init( & pver, gravit, rair, rh2o, cpair, & - zvir, latvap, latice, karman, & + zvir, latvap, latice, karman, p0_shoc, & pref_mid, nbot_shoc, ntop_shoc, & shoc_thl2tune, shoc_qw2tune, shoc_qwthl2tune, & shoc_w2tune, shoc_length_fac, shoc_c_diag_3rd_mom, & shoc_lambda_low, shoc_lambda_high, shoc_lambda_slope, & - shoc_lambda_thresh, shoc_Ckh, shoc_Ckm, shoc_Ckh_s_min, & - shoc_Ckm_s_min, shoc_Ckh_s_max, shoc_Ckm_s_max ) - + shoc_lambda_thresh, shoc_Ckh, shoc_Ckm, shoc_Ckh_s, & + shoc_Ckm_s ) + ! --------------- ! ! End ! ! Initialization ! - ! --------------- ! - - dp1 = dp1_in - - end subroutine shoc_init_e3sm - + ! --------------- ! + + dp1 = dp1_in + + end subroutine shoc_init_e3sm + ! =============================================================================== ! ! ! ! =============================================================================== ! - + subroutine shoc_tend_e3sm( & state, ptend_all, pbuf, hdtime, & cmfmc, cam_in, sgh30, & macmic_it, cld_macmic_num_steps, & dlf, det_s, det_ice, alst_o) - + !------------------------------------------------------------------- ! ! Provide tendencies of shallow convection , turbulence, and ! ! macrophysics from SHOC to E3SM ! - !------------------------------------------------------------------- ! - + !------------------------------------------------------------------- ! + use physics_types, only: physics_state, physics_ptend, & physics_state_copy, physics_ptend_init, & - physics_ptend_sum - + physics_ptend_sum + use physics_update_mod, only: physics_update use physics_buffer, only: pbuf_get_index, pbuf_old_tim_idx, pbuf_get_field, & - pbuf_set_field, physics_buffer_desc - + pbuf_set_field, physics_buffer_desc + use ppgrid, only: pver, pverp, pcols use constituents, only: cnst_get_ind use camsrfexch, only: cam_in_t - use ref_pres, only: top_lev => trop_cloud_top_lev - use time_manager, only: is_first_step + use ref_pres, only: top_lev => trop_cloud_top_lev + use time_manager, only: is_first_step use wv_saturation, only: qsat - use micro_mg_cam, only: micro_mg_version - use cldfrc2m, only: aist_vector + use micro_mg_cam, only: micro_mg_version + use cldfrc2m, only: aist_vector use trb_mtn_stress, only: compute_tms use shoc, only: shoc_main use cam_history, only: outfld use iop_data_mod, only: single_column, dp_crm - use physics_utils, only: calculate_drymmr_from_wetmmr, calculate_wetmmr_from_drymmr - + implicit none - + ! --------------- ! ! Input Auguments ! ! --------------- ! @@ -545,11 +514,11 @@ subroutine shoc_tend_e3sm( & real(r8), intent(in) :: cmfmc(pcols,pverp) ! convective mass flux--m sub c [kg/m2/s] real(r8), intent(in) :: sgh30(pcols) ! std deviation of orography [m] integer, intent(in) :: cld_macmic_num_steps ! number of mac-mic iterations - integer, intent(in) :: macmic_it ! number of mac-mic iterations + integer, intent(in) :: macmic_it ! number of mac-mic iterations ! ---------------------- ! ! Input-Output Auguments ! ! ---------------------- ! - + type(physics_buffer_desc), pointer :: pbuf(:) ! ---------------------- ! @@ -558,19 +527,18 @@ subroutine shoc_tend_e3sm( & type(physics_ptend), intent(out) :: ptend_all ! package tendencies - ! These two variables are needed for energy check + ! These two variables are needed for energy check real(r8), intent(out) :: det_s(pcols) ! Integral of detrained static energy from ice real(r8), intent(out) :: det_ice(pcols) ! Integral of detrained ice for energy check - real(r8), intent(out) :: alst_o(pcols,pver) ! H. Wang: for old liquid status fraction - + real(r8), intent(out) :: alst_o(pcols,pver) ! H. Wang: for old liquid status fraction + ! --------------- ! ! Local Variables ! ! --------------- ! - logical:: convert_back_to_wet(edsclr_dim)! To track scalars which needs a conversion back to wet mmr integer :: shoctop(pcols) - + #ifdef SHOC_SGS type(physics_state) :: state1 ! Local copy of state variable @@ -583,12 +551,11 @@ subroutine shoc_tend_e3sm( & integer :: err_code ! Diagnostic, for if some calculation goes amiss. integer :: begin_height, end_height integer :: icnt - - real(r8) :: dtime ! SHOC time step [s] - real(r8) :: edsclr_in(pcols,pver,edsclr_dim) ! Scalars to be diffused through SHOC [units vary] + + real(r8) :: dtime ! SHOC time step [s] + real(r8) :: edsclr_in(pcols,pver,edsclr_dim) ! Scalars to be diffused through SHOC [units vary] real(r8) :: edsclr_out(pcols,pver,edsclr_dim) real(r8) :: rcm_in(pcols,pver) - real(r8) :: qv_wet(pcols,pver), qv_dry(pcols,pver) ! wet [kg/kg-of-wet-air] and dry [kg/kg-of-dry-air] water vapor mmr real(r8) :: cloudfrac_shoc(pcols,pver) real(r8) :: newfice(pcols,pver) ! fraction of ice in cloud at CLUBB start [-] real(r8) :: inv_exner(pcols,pver) @@ -609,12 +576,12 @@ subroutine shoc_tend_e3sm( & real(r8) :: cloud_frac(pcols,pver) ! CLUBB cloud fraction [fraction] real(r8) :: ice_cloud_frac(pcols,pver) ! ice number aware cloud fraction, 0 or 1 real(r8) :: precipitating_ice_frac(pcols,pver) ! precipitating ice fraction, 0 or 1 - real(r8) :: liq_cloud_frac(pcols,pver) + real(r8) :: liq_cloud_frac(pcols,pver) real(r8) :: dlf2(pcols,pver) real(r8) :: isotropy(pcols,pver) real(r8) :: host_dx, host_dy real(r8) :: host_temp(pcols,pver) - real(r8) :: host_dx_in(pcols), host_dy_in(pcols) + real(r8) :: host_dx_in(pcols), host_dy_in(pcols) real(r8) :: shoc_mix_out(pcols,pver), tk_in(pcols,pver), tkh_in(pcols,pver) real(r8) :: isotropy_out(pcols,pver), tke_zt(pcols,pver) real(r8) :: w_sec_out(pcols,pver), thl_sec_out(pcols,pverp) @@ -631,115 +598,89 @@ subroutine shoc_tend_e3sm( & real(r8) :: obklen(pcols), ustar2(pcols), kinheat(pcols), kinwat(pcols) real(r8) :: dummy2(pcols), dummy3(pcols), kbfs(pcols), th(pcols,pver), thv(pcols,pver) - real(r8) :: thv2(pcols,pver) - + real(r8) :: thv2(pcols,pver) + real(r8) :: minqn, rrho(pcols,pver), rrho_i(pcols,pverp) ! minimum total cloud liquid + ice threshold [kg/kg] real(r8) :: cldthresh, frac_limit real(r8) :: ic_limit, dum1 real(r8) :: inv_exner_surf, pot_temp - + real(r8) :: wpthlp_sfc(pcols), wprtp_sfc(pcols), upwp_sfc(pcols), vpwp_sfc(pcols) real(r8) :: wtracer_sfc(pcols,edsclr_dim) - + ! Variables below are needed to compute energy integrals for conservation real(r8) :: ke_a(pcols), ke_b(pcols), te_a(pcols), te_b(pcols) real(r8) :: wv_a(pcols), wv_b(pcols), wl_b(pcols), wl_a(pcols) real(r8) :: se_dis(pcols), se_a(pcols), se_b(pcols), shoc_s(pcols,pver) real(r8) :: shoc_t(pcols,pver) - + ! --------------- ! ! Pointers ! ! --------------- ! - + real(r8), pointer, dimension(:,:) :: tke_zi ! turbulent kinetic energy, interface real(r8), pointer, dimension(:,:) :: wthv ! buoyancy flux - real(r8), pointer, dimension(:,:) :: tkh + real(r8), pointer, dimension(:,:) :: tkh real(r8), pointer, dimension(:,:) :: tk real(r8), pointer, dimension(:,:) :: cld ! cloud fraction [fraction] real(r8), pointer, dimension(:,:) :: tot_cloud_frac ! cloud fraction [fraction] real(r8), pointer, dimension(:,:) :: concld ! convective cloud fraction [fraction] real(r8), pointer, dimension(:,:) :: ast ! stratiform cloud fraction [fraction] real(r8), pointer, dimension(:,:) :: alst ! liquid stratiform cloud fraction [fraction] - real(r8), pointer, dimension(:,:) :: aist ! ice stratiform cloud fraction [fraction] - real(r8), pointer, dimension(:,:) :: cmeliq - + real(r8), pointer, dimension(:,:) :: aist ! ice stratiform cloud fraction [fraction] + real(r8), pointer, dimension(:,:) :: cmeliq + real(r8), pointer, dimension(:,:) :: qlst ! Physical in-stratus LWC [kg/kg] real(r8), pointer, dimension(:,:) :: qist ! Physical in-stratus IWC [kg/kg] real(r8), pointer, dimension(:,:) :: deepcu ! deep convection cloud fraction [fraction] - real(r8), pointer, dimension(:,:) :: shalcu ! shallow convection cloud fraction [fraction] + real(r8), pointer, dimension(:,:) :: shalcu ! shallow convection cloud fraction [fraction] real(r8), pointer, dimension(:,:) :: khzt ! eddy diffusivity on thermo levels [m^2/s] real(r8), pointer, dimension(:,:) :: khzm ! eddy diffusivity on momentum levels [m^2/s] real(r8), pointer, dimension(:) :: pblh ! planetary boundary layer height [m] - real(r8), pointer, dimension(:,:) :: dp_icwmr ! deep convection in cloud mixing ratio [kg/kg] - real(r8), pointer, dimension(:,:) :: cmfmc_sh ! Shallow convective mass flux--m subc (pcols,pverp) [kg/m2/s/] + real(r8), pointer, dimension(:,:) :: dp_icwmr ! deep convection in cloud mixing ratio [kg/kg] + real(r8), pointer, dimension(:,:) :: cmfmc_sh ! Shallow convective mass flux--m subc (pcols,pverp) [kg/m2/s/] - real(r8), pointer, dimension(:,:) :: prer_evap + real(r8), pointer, dimension(:,:) :: prer_evap real(r8), pointer, dimension(:,:) :: accre_enhan real(r8), pointer, dimension(:,:) :: relvar - + logical :: lqice(pcnst) real(r8) :: relvarmax - + !------------------------------------------------------------------! !------------------------------------------------------------------! !------------------------------------------------------------------! ! MAIN COMPUTATION BEGINS HERE ! !------------------------------------------------------------------! !------------------------------------------------------------------! - !------------------------------------------------------------------! - + !------------------------------------------------------------------! + ! Get indicees for cloud and ice mass and cloud and ice number ic_limit = 1.e-12_r8 frac_limit = 0.01_r8 - + call cnst_get_ind('CLDLIQ',ixcldliq) call cnst_get_ind('CLDICE',ixcldice) call cnst_get_ind('NUMLIQ',ixnumliq) call cnst_get_ind('NUMICE',ixnumice) - + call physics_ptend_init(ptend_loc,state%psetcols, 'shoc', ls=.true., lu=.true., lv=.true., lq=lq) - + call physics_state_copy(state,state1) - + ! Determine number of columns and which chunk computation is to be performed on ncol = state%ncol - lchnk = state%lchnk - - !obtain wet mmr from the state vector - qv_wet (:,:) = state1%q(:,:,ixq) - icnt = 0 - do ixind = 1, pcnst - if (lq(ixind)) then - icnt = icnt + 1 - - !Track which scalars need a conversion to wetmmr after SHOC main call - convert_back_to_wet(icnt) = .false. - - if(lq_dry_wet_cnvr(ixind)) then !convert from wet to dry mmr if true - convert_back_to_wet(icnt) = .true. - !--------------------------------------------------------------------------------------- - !Wet to dry mixing ratios: - !------------------------- - !Since state scalars from the host model are wet mixing ratios and SHOC needs these - !scalars in dry mixing ratios, we convert the wet mixing ratios to dry mixing ratio - !if lq_dry_wet_cnvr is .true. for that scalar - !NOTE:Function calculate_drymmr_from_wetmmr takes 2 arguments: (wet mmr and "wet" water - !vapor mixing ratio) - !--------------------------------------------------------------------------------------- - state1%q(:,:,ixind) = calculate_drymmr_from_wetmmr(ncol, pver,state1%q(:,:,ixind), qv_wet) - endif - endif - enddo + lchnk = state%lchnk + + ! Determine time step of physics buffer + itim_old = pbuf_old_tim_idx() - ! Determine time step of physics buffer - itim_old = pbuf_old_tim_idx() - - ! Establish associations between pointers and physics buffer fields + ! Establish associations between pointers and physics buffer fields call pbuf_get_field(pbuf, tke_idx, tke_zi) - call pbuf_get_field(pbuf, wthv_idx, wthv, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) - call pbuf_get_field(pbuf, tkh_idx, tkh, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) - call pbuf_get_field(pbuf, tk_idx, tk, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) + call pbuf_get_field(pbuf, wthv_idx, wthv, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) + call pbuf_get_field(pbuf, tkh_idx, tkh, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) + call pbuf_get_field(pbuf, tk_idx, tk, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) call pbuf_get_field(pbuf, cld_idx, cld, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) call pbuf_get_field(pbuf, tot_cloud_frac_idx, tot_cloud_frac, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) call pbuf_get_field(pbuf, concld_idx, concld, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) @@ -748,7 +689,7 @@ subroutine shoc_tend_e3sm( & call pbuf_get_field(pbuf, aist_idx, aist, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) call pbuf_get_field(pbuf, qlst_idx, qlst, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) call pbuf_get_field(pbuf, qist_idx, qist, start=(/1,1,itim_old/), kount=(/pcols,pver,1/)) - + call pbuf_get_field(pbuf, prer_evap_idx, prer_evap) call pbuf_get_field(pbuf, accre_enhan_idx, accre_enhan) call pbuf_get_field(pbuf, cmeliq_idx, cmeliq) @@ -759,40 +700,40 @@ subroutine shoc_tend_e3sm( & call pbuf_get_field(pbuf, kvh_idx, khzt) call pbuf_get_field(pbuf, pblh_idx, pblh) call pbuf_get_field(pbuf, icwmrdp_idx, dp_icwmr) - call pbuf_get_field(pbuf, cmfmc_sh_idx, cmfmc_sh) - + call pbuf_get_field(pbuf, cmfmc_sh_idx, cmfmc_sh) + ! Determine SHOC time step. - + dtime = shoc_timestep - + ! If requested SHOC timestep is < 0 then set the SHOC time step ! equal to hdtime (the macrophysics/microphysics timestep). - + if (dtime < 0._r8) then dtime = hdtime endif - + ! Now perform checks to determine if the requested SHOC timestep ! is reasonable based on the host model time step. - + ! Is SHOC timestep greater than the macrophysics/microphysics timestep? if (dtime .gt. hdtime) then call endrun('shoc_tend_e3sm: Requested SHOC time step is greater than the macrophysics/microphysics timestep') endif - + ! Does SHOC timestep divide evenly into the macrophysics/microphyscs timestep? if (mod(hdtime,dtime) .ne. 0) then call endrun('shoc_tend_e3sm: SHOC time step and HOST time step NOT compatible') endif ! If we survived this far, then the SHOC timestep is valid. - - ! determine number of timesteps SHOC core should be advanced, - ! host time step divided by SHOC time step + + ! determine number of timesteps SHOC core should be advanced, + ! host time step divided by SHOC time step nadv = max(hdtime/dtime,1._r8) ! Set grid space, in meters. If SCM, set to a grid size representative - ! of a typical GCM. Otherwise, compute locally. + ! of a typical GCM. Otherwise, compute locally. if (single_column .and. .not. dp_crm) then host_dx_in(:) = 100000._r8 host_dy_in(:) = 100000._r8 @@ -803,55 +744,55 @@ subroutine shoc_tend_e3sm( & else call grid_size(state1, host_dx_in, host_dy_in) endif - + minqn = 0._r8 newfice(:,:) = 0._r8 where(state1%q(:ncol,:pver,ixcldice) .gt. minqn) & - newfice(:ncol,:pver) = state1%q(:ncol,:pver,ixcldice)/(state1%q(:ncol,:pver,ixcldliq)+state1%q(:ncol,:pver,ixcldice)) - + newfice(:ncol,:pver) = state1%q(:ncol,:pver,ixcldice)/(state1%q(:ncol,:pver,ixcldliq)+state1%q(:ncol,:pver,ixcldice)) + ! TODO: Create a general function to calculate Exner's formula - see full ! comment in micro_p3_interface.F90 do k=1,pver do i=1,ncol inv_exner(i,k) = 1._r8/((state1%pmid(i,k)/p0_shoc)**(rair/cpair)) enddo - enddo - - ! At each SHOC call, initialize mean momentum and thermo SHOC state + enddo + + ! At each SHOC call, initialize mean momentum and thermo SHOC state ! from the E3SM state - + do k=1,pver ! loop over levels do i=1,ncol ! loop over columns - + rvm(i,k) = state1%q(i,k,ixq) rcm(i,k) = state1%q(i,k,ixcldliq) rtm(i,k) = rvm(i,k) + rcm(i,k) um(i,k) = state1%u(i,k) vm(i,k) = state1%v(i,k) - + pot_temp = state1%t(i,k)*inv_exner(i,k) thlm(i,k) = pot_temp-(pot_temp/state1%t(i,k))*(latvap/cpair)*state1%q(i,k,ixcldliq) - thv(i,k) = state1%t(i,k)*inv_exner(i,k)*(1.0_r8+zvir*state1%q(i,k,ixq)-state1%q(i,k,ixcldliq)) - + thv(i,k) = state1%t(i,k)*inv_exner(i,k)*(1.0_r8+zvir*state1%q(i,k,ixq)-state1%q(i,k,ixcldliq)) + tke_zt(i,k) = max(tke_tol,state1%q(i,k,ixtke)) - - ! Cloud fraction needs to be initialized for first + + ! Cloud fraction needs to be initialized for first ! PBL height calculation call - cloud_frac(i,k) = alst(i,k) - + cloud_frac(i,k) = alst(i,k) + enddo - enddo - + enddo + ! ------------------------------------------------- ! ! Prepare inputs for SHOC call ! - ! ------------------------------------------------- ! - + ! ------------------------------------------------- ! + do k=1,pver do i=1,ncol dz_g(i,k) = state1%zi(i,k)-state1%zi(i,k+1) ! compute thickness enddo enddo - + ! Define the SHOC thermodynamic grid (in units of m) wm_zt(:,pver) = 0._r8 do k=1,pver @@ -862,7 +803,7 @@ subroutine shoc_tend_e3sm( & shoc_s(i,k) = state1%s(i,k) enddo enddo - + do k=1,pverp do i=1,ncol zi_g(i,k) = state1%zi(i,k)-state1%zi(i,pver+1) @@ -883,14 +824,14 @@ subroutine shoc_tend_e3sm( & wprtp_sfc(i) = cam_in%cflx(i,1)/(rrho_i(i,pverp)) ! Latent heat flux upwp_sfc(i) = cam_in%wsx(i)/rrho_i(i,pverp) ! Surface meridional momentum flux - vpwp_sfc(i) = cam_in%wsy(i)/rrho_i(i,pverp) ! Surface zonal momentum flux + vpwp_sfc(i) = cam_in%wsy(i)/rrho_i(i,pverp) ! Surface zonal momentum flux wtracer_sfc(i,:) = 0._r8 ! in E3SM tracer fluxes are done elsewhere - enddo - - ! Do the same for tracers + enddo + + ! Do the same for tracers icnt=0 do ixind=1,pcnst - if (lq(ixind)) then + if (lq(ixind)) then icnt=icnt+1 do k=1,pver do i=1,ncol @@ -898,67 +839,43 @@ subroutine shoc_tend_e3sm( & enddo enddo end if - enddo - + enddo + ! ------------------------------------------------- ! ! Actually call SHOC ! - ! ------------------------------------------------- ! + ! ------------------------------------------------- ! call shoc_main( & ncol, pver, pverp, dtime, nadv, & ! Input - host_dx_in(:ncol), host_dy_in(:ncol), thv(:ncol,:),& ! Input + host_dx_in(:ncol), host_dy_in(:ncol), thv(:ncol,:),& ! Input zt_g(:ncol,:), zi_g(:ncol,:), state%pmid(:ncol,:pver), state%pint(:ncol,:pverp), state1%pdel(:ncol,:pver),& ! Input - wpthlp_sfc(:ncol), wprtp_sfc(:ncol), upwp_sfc(:ncol), vpwp_sfc(:ncol), & ! Input - wtracer_sfc(:ncol,:), edsclr_dim, wm_zt(:ncol,:), & ! Input - inv_exner(:ncol,:),state1%phis(:ncol), & ! Input - shoc_s(:ncol,:), tke_zt(:ncol,:), thlm(:ncol,:), rtm(:ncol,:), & ! Input/Ouput - um(:ncol,:), vm(:ncol,:), edsclr_in(:ncol,:,:), & ! Input/Output - wthv(:ncol,:),tkh(:ncol,:),tk(:ncol,:), & ! Input/Output - rcm(:ncol,:),cloud_frac(:ncol,:), & ! Input/Output + wpthlp_sfc(:ncol), wprtp_sfc(:ncol), upwp_sfc(:ncol), vpwp_sfc(:ncol), & ! Input + wtracer_sfc(:ncol,:), edsclr_dim, wm_zt(:ncol,:), & ! Input + inv_exner(:ncol,:),state1%phis(:ncol), & ! Input + shoc_s(:ncol,:), tke_zt(:ncol,:), thlm(:ncol,:), rtm(:ncol,:), & ! Input/Ouput + um(:ncol,:), vm(:ncol,:), edsclr_in(:ncol,:,:), & ! Input/Output + wthv(:ncol,:),tkh(:ncol,:),tk(:ncol,:), & ! Input/Output + rcm(:ncol,:),cloud_frac(:ncol,:), & ! Input/Output pblh(:ncol), & ! Output shoc_mix_out(:ncol,:), isotropy_out(:ncol,:), & ! Output (diagnostic) - w_sec_out(:ncol,:), thl_sec_out(:ncol,:), qw_sec_out(:ncol,:), qwthl_sec_out(:ncol,:), & ! Output (diagnostic) + w_sec_out(:ncol,:), thl_sec_out(:ncol,:), qw_sec_out(:ncol,:), qwthl_sec_out(:ncol,:), & ! Output (diagnostic) wthl_sec_out(:ncol,:), wqw_sec_out(:ncol,:), wtke_sec_out(:ncol,:), & ! Output (diagnostic) uw_sec_out(:ncol,:), vw_sec_out(:ncol,:), w3_out(:ncol,:), & ! Output (diagnostic) wqls_out(:ncol,:),brunt_out(:ncol,:),rcm2(:ncol,:)) ! Output (diagnostic) - + ! Transfer back to pbuf variables - + do k=1,pver - do i=1,ncol + do i=1,ncol cloud_frac(i,k) = min(cloud_frac(i,k),1._r8) enddo enddo - - !obtain water vapor mmr which is a "dry" mmr at this point from the SHOC output - qv_dry(:,:) = edsclr_in(:,:,ixq) - !---------------- - !DRY-TO-WET MMRs: - !---------------- - !Since the host model needs wet mixing ratio tendencies(state vector has wet mixing ratios), - !we need to convert dry mixing ratios from SHOC to wet mixing ratios before extracting tendencies - !NOTE:Function calculate_wetmmr_from_drymmr takes 2 arguments: (wet mmr and "dry" water vapor - !mixing ratio) - do ixind=1,edsclr_dim - if(convert_back_to_wet(ixind)) then - edsclr_out(:,:,ixind) = calculate_wetmmr_from_drymmr(ncol, pver, edsclr_in(:,:,ixind), qv_dry) - else - edsclr_out(:,:,ixind) = edsclr_in(:,:,ixind) - endif - enddo - !convert state1%q to wet mixing ratios - qv_dry(:,:) = state1%q(:,:,ixq) - icnt = 0 - do ixind = 1, pcnst - if (lq(ixind)) then - icnt = icnt + 1 - if(convert_back_to_wet(icnt)) then !convert from wet to dry mmr if true - state1%q(:,:,ixind) = calculate_wetmmr_from_drymmr(ncol, pver, state1%q(:,:,ixind), qv_dry) - endif - endif - enddo - rcm(:,:) = calculate_wetmmr_from_drymmr(ncol, pver, rcm, qv_dry) - rtm(:,:) = calculate_wetmmr_from_drymmr(ncol, pver, rtm, qv_dry) + +!sort out edsclr_in, edsclr_out + do ixind=1,edsclr_dim + edsclr_out(:,:,ixind) = edsclr_in(:,:,ixind) + enddo + ! Eddy diffusivities and TKE are needed for aerosol activation code. ! Linearly interpolate from midpoint grid and onto the interface grid. @@ -974,17 +891,17 @@ subroutine shoc_tend_e3sm( & ! Now compute the tendencies of SHOC to E3SM do k=1,pver do i=1,ncol - + ptend_loc%u(i,k) = (um(i,k)-state1%u(i,k))/hdtime - ptend_loc%v(i,k) = (vm(i,k)-state1%v(i,k))/hdtime + ptend_loc%v(i,k) = (vm(i,k)-state1%v(i,k))/hdtime ptend_loc%q(i,k,ixq) = (rtm(i,k)-rcm(i,k)-state1%q(i,k,ixq))/hdtime ! water vapor ptend_loc%q(i,k,ixcldliq) = (rcm(i,k)-state1%q(i,k,ixcldliq))/hdtime ! Tendency of liquid water ptend_loc%s(i,k) = (shoc_s(i,k)-state1%s(i,k))/hdtime - + ptend_loc%q(i,k,ixtke)=(tke_zt(i,k)-state1%q(i,k,ixtke))/hdtime ! TKE - + ! Apply tendencies to ice mixing ratio, liquid and ice number, and aerosol constituents. - ! Loading up this array doesn't mean the tendencies are applied. + ! Loading up this array doesn't mean the tendencies are applied. ! edsclr_out is compressed with just the constituents being used, ptend and state are not compressed icnt=0 @@ -992,45 +909,48 @@ subroutine shoc_tend_e3sm( & if (lq(ixind)) then icnt=icnt+1 if ((ixind /= ixq) .and. (ixind /= ixcldliq) .and. (ixind /= ixtke)) then - ptend_loc%q(i,k,ixind) = (edsclr_out(i,k,icnt)-state1%q(i,k,ixind))/hdtime ! transported constituents + ptend_loc%q(i,k,ixind) = (edsclr_out(i,k,icnt)-state1%q(i,k,ixind))/hdtime ! transported constituents end if end if enddo enddo - enddo - + enddo + cmeliq(:,:) = ptend_loc%q(:,:,ixcldliq) - + ! Update physics tendencies call physics_ptend_init(ptend_all, state%psetcols, 'shoc') call physics_ptend_sum(ptend_loc,ptend_all,ncol) call physics_update(state1,ptend_loc,hdtime) - + + ! ------------------------------------------------------------ ! ! ------------------------------------------------------------ ! - ! ------------------------------------------------------------ ! ! ------------------------------------------------------------ ! ! The rest of the code deals with diagnosing variables ! ! for microphysics/radiation computation and macrophysics ! ! ------------------------------------------------------------ ! ! ------------------------------------------------------------ ! - ! ------------------------------------------------------------ ! - - ! --------------------------------------------------------------------------------- ! + ! ------------------------------------------------------------ ! + + ! --------------------------------------------------------------------------------- ! ! COMPUTE THE ICE CLOUD DETRAINMENT ! ! Detrainment of convective condensate into the environment or stratiform cloud ! ! --------------------------------------------------------------------------------- ! - + ! Initialize the shallow convective detrainment rate, will always be zero dlf2(:,:) = 0.0_r8 + det_ice(:)=0.0 + det_s(:)=0.0 + lqice(:) = .false. lqice(ixcldliq) = .true. lqice(ixcldice) = .true. lqice(ixnumliq) = .true. - lqice(ixnumice) = .true. - - call physics_ptend_init(ptend_loc,state%psetcols, 'clubb_det', ls=.true., lq=lqice) + lqice(ixnumice) = .true. + + call physics_ptend_init(ptend_loc,state%psetcols, 'clubb_det', ls=.true., lq=lqice) do k=1,pver do i=1,ncol if( state1%t(i,k) > shoc_tk1 ) then @@ -1042,62 +962,62 @@ subroutine shoc_tend_e3sm( & !(clubb_tk1 - clubb_tk2) is also 30.0 but it introduced a non-bfb change dum1 = ( shoc_tk1 - state1%t(i,k) ) /(shoc_tk1 - shoc_tk2) endif - + ptend_loc%q(i,k,ixcldliq) = dlf(i,k) * ( 1._r8 - dum1 ) ptend_loc%q(i,k,ixcldice) = dlf(i,k) * dum1 ptend_loc%q(i,k,ixnumliq) = 3._r8 * ( max(0._r8, ( dlf(i,k) - dlf2(i,k) )) * ( 1._r8 - dum1 ) ) & / (4._r8*3.14_r8* shoc_liq_deep**3*997._r8) + & ! Deep Convection 3._r8 * ( dlf2(i,k) * ( 1._r8 - dum1 ) ) & - / (4._r8*3.14_r8*shoc_liq_sh**3*997._r8) ! Shallow Convection + / (4._r8*3.14_r8*shoc_liq_sh**3*997._r8) ! Shallow Convection ptend_loc%q(i,k,ixnumice) = 3._r8 * ( max(0._r8, ( dlf(i,k) - dlf2(i,k) )) * dum1 ) & / (4._r8*3.14_r8*shoc_ice_deep**3*500._r8) + & ! Deep Convection 3._r8 * ( dlf2(i,k) * dum1 ) & / (4._r8*3.14_r8*shoc_ice_sh**3*500._r8) ! Shallow Convection ptend_loc%s(i,k) = dlf(i,k) * dum1 * latice - + ! Only rliq is saved from deep convection, which is the reserved liquid. We need to keep ! track of the integrals of ice and static energy that is effected from conversion to ice ! so that the energy checker doesn't complain. det_s(i) = det_s(i) + ptend_loc%s(i,k)*state1%pdel(i,k)/gravit det_ice(i) = det_ice(i) - ptend_loc%q(i,k,ixcldice)*state1%pdel(i,k)/gravit - + enddo enddo det_ice(:ncol) = det_ice(:ncol)/1000._r8 ! divide by density of water - + call physics_ptend_sum(ptend_loc,ptend_all,ncol) call physics_update(state1,ptend_loc,hdtime) - + ! For purposes of this implementation, just set relvar and accre_enhan to 1 relvar(:,:) = 1.0_r8 - accre_enhan(:,:) = 1._r8 - + accre_enhan(:,:) = 1._r8 + ! +++ JShpund: add relative cloud liquid variance (a vectorized version based on CLUBB) ! TODO: double check the hardcoded values ('relvarmax', '0.001_r8') relvarmax = 10.0_r8 where (rcm(:ncol,:pver) /= 0.0 .and. rcm2(:ncol,:pver) /= 0.0) & relvar(:ncol,:pver) = min(relvarmax,max(0.001_r8,rcm(:ncol,:pver)**2.0/rcm2(:ncol,:pver))) - ! --------------------------------------------------------------------------------- ! + ! --------------------------------------------------------------------------------- ! ! Diagnose some quantities that are computed in macrop_tend here. ! ! These are inputs required for the microphysics calculation. ! ! ! ! FIRST PART COMPUTES THE STRATIFORM CLOUD FRACTION FROM SHOC CLOUD FRACTION ! - ! --------------------------------------------------------------------------------- ! - + ! --------------------------------------------------------------------------------- ! + ! HW: set alst to alst_o before getting updated if(liqcf_fix) then if(.not.is_first_step()) alst_o(:ncol,:pver) = alst(:ncol,:pver) endif - ! initialize variables + ! initialize variables alst(:,:) = 0.0_r8 - qlst(:,:) = 0.0_r8 - + qlst(:,:) = 0.0_r8 + do k=1,pver do i=1,ncol - alst(i,k) = cloud_frac(i,k) + alst(i,k) = cloud_frac(i,k) qlst(i,k) = rcm(i,k)/max(0.01_r8,alst(i,k)) ! Incloud stratus condensate mixing ratio enddo enddo @@ -1105,56 +1025,56 @@ subroutine shoc_tend_e3sm( & ! HW if(liqcf_fix) then if(is_first_step()) alst_o(:ncol,:pver) = alst(:ncol,:pver) - endif - - ! --------------------------------------------------------------------------------- ! + endif + + ! --------------------------------------------------------------------------------- ! ! THIS PART COMPUTES CONVECTIVE AND DEEP CONVECTIVE CLOUD FRACTION ! - ! --------------------------------------------------------------------------------- ! - + ! --------------------------------------------------------------------------------- ! + deepcu(:,pver) = 0.0_r8 shalcu(:,pver) = 0.0_r8 - + do k=1,pver-1 do i=1,ncol - ! diagnose the deep convective cloud fraction, as done in macrophysics based on the - ! deep convective mass flux, read in from pbuf. Since shallow convection is never + ! diagnose the deep convective cloud fraction, as done in macrophysics based on the + ! deep convective mass flux, read in from pbuf. Since shallow convection is never ! called, the shallow convective mass flux will ALWAYS be zero, ensuring that this cloud - ! fraction is purely from deep convection scheme. + ! fraction is purely from deep convection scheme. deepcu(i,k) = max(0.0_r8,min(dp1*log(1.0_r8+500.0_r8*(cmfmc(i,k+1)-cmfmc_sh(i,k+1))),0.6_r8)) shalcu(i,k) = 0._r8 - + if (deepcu(i,k) <= frac_limit .or. dp_icwmr(i,k) < ic_limit) then deepcu(i,k) = 0._r8 endif - - ! using the deep convective cloud fraction, and SHOC cloud fraction (variable + + ! using the deep convective cloud fraction, and SHOC cloud fraction (variable ! "cloud_frac"), compute the convective cloud fraction. This follows the formulation - ! found in macrophysics code. Assumes that convective cloud is all nonstratiform cloud + ! found in macrophysics code. Assumes that convective cloud is all nonstratiform cloud ! from SHOC plus the deep convective cloud fraction concld(i,k) = min(cloud_frac(i,k)-alst(i,k)+deepcu(i,k),0.80_r8) enddo - enddo - - ! --------------------------------------------------------------------------------- ! + enddo + + ! --------------------------------------------------------------------------------- ! ! COMPUTE THE ICE CLOUD FRACTION PORTION ! ! use the aist_vector function to compute the ice cloud fraction ! ! --------------------------------------------------------------------------------- ! - + do k=1,pver call aist_vector(state1%q(:,k,ixq),state1%t(:,k),state1%pmid(:,k),state1%q(:,k,ixcldice), & state1%q(:,k,ixnumice),cam_in%landfrac(:),cam_in%snowhland(:),aist(:,k),ncol) enddo - - ! --------------------------------------------------------------------------------- ! + + ! --------------------------------------------------------------------------------- ! ! THIS PART COMPUTES THE LIQUID STRATUS FRACTION ! ! ! ! For now leave the computation of ice stratus fraction from macrop_driver intact ! - ! because SHOC does nothing with ice. Here I simply overwrite the liquid stratus ! + ! because SHOC does nothing with ice. Here I simply overwrite the liquid stratus ! ! fraction that was coded in macrop_driver ! - ! --------------------------------------------------------------------------------- ! - + ! --------------------------------------------------------------------------------- ! + ! Recompute net stratus fraction using maximum over-lapping assumption, as done - ! in macrophysics code, using alst computed above and aist read in from physics buffer + ! in macrophysics code, using alst computed above and aist read in from physics buffer cldthresh=1.e-18_r8 @@ -1163,27 +1083,27 @@ subroutine shoc_tend_e3sm( & ast(i,k) = max(alst(i,k),aist(i,k)) - qist(i,k) = state1%q(i,k,ixcldice)/max(0.01_r8,aist(i,k)) + qist(i,k) = state1%q(i,k,ixcldice)/max(0.01_r8,aist(i,k)) enddo enddo - - ! Probably need to add deepcu cloud fraction to the cloud fraction array, else would just + + ! Probably need to add deepcu cloud fraction to the cloud fraction array, else would just ! be outputting the shallow convective cloud fraction - + ! Add liq, ice, and precipitating ice fractions here. These are purely ! diagnostic outputs and do not impact the rest of the code. The qi threshold for - ! setting ice_cloud_fraction and the qi dependent ni_threshold are tunable. + ! setting ice_cloud_fraction and the qi dependent ni_threshold are tunable. liq_cloud_frac = 0.0_r8 ice_cloud_frac = 0.0_r8 precipitating_ice_frac = 0.0_r8 tot_cloud_frac = 0.0_r8 - + do k=1,pver do i=1,ncol cloud_frac(i,k) = min(ast(i,k)+deepcu(i,k),1.0_r8) liq_cloud_frac(i,k) = alst(i,k) - if (state1%q(i,k,ixcldice) .ge. 1.0e-5_r8) then + if (state1%q(i,k,ixcldice) .ge. 1.0e-5_r8) then if (state1%q(i,k,ixnumice) .ge. state1%q(i,k,ixcldice)*5.0e7_r8) then ice_cloud_frac(i,k) = 1.0_r8 else @@ -1193,8 +1113,8 @@ subroutine shoc_tend_e3sm( & tot_cloud_frac(i,k) = min(1.0_r8, max(ice_cloud_frac(i,k),liq_cloud_frac(i,k))+deepcu(i,k)) enddo enddo - - cld(:,1:pver) = cloud_frac(:,1:pver) + + cld(:,1:pver) = cloud_frac(:,1:pver) ! --------------------------------------------------------! ! Output fields @@ -1203,14 +1123,14 @@ subroutine shoc_tend_e3sm( & do k=1,pverp do i=1,ncol wthl_output(i,k) = wthl_sec_out(i,k) * rrho_i(i,k) * cpair - wqw_output(i,k) = wqw_sec_out(i,k) * rrho_i(i,k) * latvap + wqw_output(i,k) = wqw_sec_out(i,k) * rrho_i(i,k) * latvap enddo enddo do k=1,pver do i=1,ncol wthv_output(i,k) = wthv(i,k) * rrho(i,k) * cpair - wql_output(i,k) = wqls_out(i,k) * rrho(i,k) * latvap + wql_output(i,k) = wqls_out(i,k) * rrho(i,k) * latvap enddo enddo @@ -1239,11 +1159,12 @@ subroutine shoc_tend_e3sm( & call outfld('PRECIPITATING_ICE_FRAC',precipitating_ice_frac,pcols,lchnk) call outfld('LIQ_CLOUD_FRAC',liq_cloud_frac,pcols,lchnk) call outfld('TOT_CLOUD_FRAC',tot_cloud_frac,pcols,lchnk) + call outfld('PBLH',pblh,pcols,lchnk) + +#endif + return + end subroutine shoc_tend_e3sm -#endif - return - end subroutine shoc_tend_e3sm - subroutine grid_size(state, grid_dx, grid_dy) ! Determine the size of the grid for each of the columns in state @@ -1251,11 +1172,11 @@ subroutine grid_size(state, grid_dx, grid_dy) use shr_const_mod, only: shr_const_pi use physics_types, only: physics_state use ppgrid, only: pver, pverp, pcols - + type(physics_state), intent(in) :: state real(r8), intent(out) :: grid_dx(pcols), grid_dy(pcols) ! E3SM grid [m] - real(r8), parameter :: earth_ellipsoid1 = 111132.92_r8 ! World Geodetic System 1984 (WGS84) + real(r8), parameter :: earth_ellipsoid1 = 111132.92_r8 ! World Geodetic System 1984 (WGS84) ! first coefficient, meters per degree longitude at equator real(r8), parameter :: earth_ellipsoid2 = 559.82_r8 ! second expansion coefficient for WGS84 ellipsoid real(r8), parameter :: earth_ellipsoid3 = 1.175_r8 ! third expansion coefficient for WGS84 ellipsoid @@ -1271,29 +1192,29 @@ subroutine grid_size(state, grid_dx, grid_dy) ! convert latitude to radians lat_in_rad = state%lat(i)*(shr_const_pi/180._r8) - + ! Now find meters per degree latitude ! Below equation finds distance between two points on an ellipsoid, derived from expansion - ! taking into account ellipsoid using World Geodetic System (WGS84) reference + ! taking into account ellipsoid using World Geodetic System (WGS84) reference mpdeglat = earth_ellipsoid1 - earth_ellipsoid2 * cos(2._r8*lat_in_rad) + earth_ellipsoid3 * cos(4._r8*lat_in_rad) grid_dx(i) = mpdeglat * degree grid_dy(i) = grid_dx(i) ! Assume these are the same - enddo + enddo + + end subroutine grid_size - end subroutine grid_size - subroutine grid_size_planar_uniform(grid_dx, grid_dy) - + ! Get size of grid box if in doubly period planar mode ! At time of implementation planar dycore only supports uniform grids. - + use iop_data_mod, only: dyn_dx_size - + real(r8), intent(out) :: grid_dx, grid_dy grid_dx = dyn_dx_size grid_dy = grid_dx - + end subroutine grid_size_planar_uniform end module shoc_intr diff --git a/components/eam/src/physics/cam/trb_mtn_stress.F90 b/components/eam/src/physics/cam/trb_mtn_stress.F90 index ff730872e4a8..3698ddcfeb3a 100644 --- a/components/eam/src/physics/cam/trb_mtn_stress.F90 +++ b/components/eam/src/physics/cam/trb_mtn_stress.F90 @@ -1,7 +1,16 @@ +! Include bit-for-bit math macros. +#include "bfb_math.inc" + module trb_mtn_stress + ! Bit-for-bit math functions. +#ifdef SCREAM_CONFIG_IS_CMAKE + use physics_share_f2c, only: scream_pow, scream_sqrt, scream_cbrt, scream_gamma, scream_log, & + scream_log10, scream_exp, scream_erf +#endif + implicit none - private + private save public init_tms ! Initialization @@ -17,7 +26,7 @@ module trb_mtn_stress real(r8), parameter :: z0max = 100._r8 ! Maximum value of z_0 for orography [ m ] real(r8), parameter :: dv2min = 0.01_r8 ! Minimum shear squared [ m2/s2 ] real(r8) :: orocnst ! Converts from standard deviation to height [ no unit ] - real(r8) :: z0fac ! Factor determining z_0 from orographic standard deviation [ no unit ] + real(r8) :: z0fac ! Factor determining z_0 from orographic standard deviation [ no unit ] real(r8) :: karman ! von Karman constant real(r8) :: gravit ! Acceleration due to gravity real(r8) :: rair ! Gas constant for dry air @@ -49,7 +58,7 @@ subroutine init_tms( kind, oro_in, z0fac_in, karman_in, gravit_in, rair_in, & karman = karman_in gravit = gravit_in rair = rair_in - + end subroutine init_tms !============================================================================ ! @@ -58,11 +67,11 @@ end subroutine init_tms subroutine compute_tms( pcols , pver , ncol , & u , v , t , pmid , exner , & - zm , sgh , ksrf , taux , tauy , & + zm , sgh , ksrf , taux , tauy , & landfrac ) !------------------------------------------------------------------------------ ! - ! Turbulent mountain stress parameterization ! + ! Turbulent mountain stress parameterization ! ! ! ! Returns surface drag coefficient and stress associated with subgrid mountains ! ! For points where the orographic variance is small ( including ocean ), ! @@ -72,7 +81,7 @@ subroutine compute_tms( pcols , pver , ncol , & !------------------------------------------------------------------------------ ! ! ---------------------- ! - ! Input-Output Arguments ! + ! Input-Output Arguments ! ! ---------------------- ! integer, intent(in) :: pcols ! Number of columns dimensioned @@ -87,7 +96,7 @@ subroutine compute_tms( pcols , pver , ncol , & real(r8), intent(in) :: zm(pcols,pver) ! Layer mid-point height [ m ] real(r8), intent(in) :: sgh(pcols) ! Standard deviation of orography [ m ] real(r8), intent(in) :: landfrac(pcols) ! Land fraction [ fraction ] - + real(r8), intent(out) :: ksrf(pcols) ! Surface drag coefficient [ kg/s/m2 ] real(r8), intent(out) :: taux(pcols) ! Surface zonal wind stress [ N/m2 ] real(r8), intent(out) :: tauy(pcols) ! Surface meridional wind stress [ N/m2 ] @@ -98,7 +107,7 @@ subroutine compute_tms( pcols , pver , ncol , & integer :: i ! Loop index integer :: kb, kt ! Bottom and top of source region - + real(r8) :: horo ! Orographic height [ m ] real(r8) :: z0oro ! Orographic z0 for momentum [ m ] real(r8) :: dv2 ! (delta v)**2 [ m2/s2 ] @@ -111,7 +120,7 @@ subroutine compute_tms( pcols , pver , ncol , & ! ----------------------- ! ! Main Computation Begins ! ! ----------------------- ! - + do i = 1, ncol ! determine subgrid orgraphic height ( mean to peak ) @@ -134,19 +143,19 @@ subroutine compute_tms( pcols , pver , ncol , & ! Calculate neutral drag coefficient - cd = ( karman / log( ( zm(i,pver) + z0oro ) / z0oro) )**2 + cd = bfb_square( karman / bfb_log( ( zm(i,pver) + z0oro ) / z0oro) ) ! Calculate the Richardson number over the lowest 2 layers kt = pver - 1 kb = pver - dv2 = max( ( u(i,kt) - u(i,kb) )**2 + ( v(i,kt) - v(i,kb) )**2, dv2min ) + dv2 = max( bfb_square( u(i,kt) - u(i,kb) ) + bfb_square( v(i,kt) - v(i,kb) ), dv2min ) ! Modification : Below computation of Ri is wrong. Note that 'Exner' function here is ! inverse exner function. Here, exner function is not multiplied in ! the denominator. Also, we should use moist Ri not dry Ri. ! Also, this approach using the two lowest model layers can be potentially - ! sensitive to the vertical resolution. + ! sensitive to the vertical resolution. ! OK. I only modified the part associated with exner function. ri = 2._r8 * gravit * ( t(i,kt) * exner(i,kt) - t(i,kb) * exner(i,kb) ) * ( zm(i,kt) - zm(i,kb) ) & @@ -156,7 +165,7 @@ subroutine compute_tms( pcols , pver , ncol , & ! / ( ( t(i,kt) + t(i,kb) ) * dv2 ) ! Calculate the instability function and modify the neutral drag cofficient. - ! We should probably follow more elegant approach like Louis et al (1982) or Bretherton and Park (2009) + ! We should probably follow more elegant approach like Louis et al (1982) or Bretherton and Park (2009) ! but for now we use very crude approach : just 1 for ri < 0, 0 for ri > 1, and linear ramping. stabfri = max( 0._r8, min( 1._r8, 1._r8 - ri ) ) @@ -164,8 +173,8 @@ subroutine compute_tms( pcols , pver , ncol , & ! Compute density, velocity magnitude and stress using bottom level properties - rho = pmid(i,pver) / ( rair * t(i,pver) ) - vmag = sqrt( u(i,pver)**2 + v(i,pver)**2 ) + rho = pmid(i,pver) / ( rair * t(i,pver) ) + vmag = bfb_sqrt( bfb_square(u(i,pver)) + bfb_square(v(i,pver)) ) ksrf(i) = rho * cd * vmag * landfrac(i) taux(i) = -ksrf(i) * u(i,pver) tauy(i) = -ksrf(i) * v(i,pver) @@ -173,7 +182,7 @@ subroutine compute_tms( pcols , pver , ncol , & end if end do - + return end subroutine compute_tms diff --git a/components/eam/src/physics/cam/wv_sat_scream.F90 b/components/eam/src/physics/cam/wv_sat_scream.F90 index d52e7e5cc30b..b43ae6bfcc3c 100644 --- a/components/eam/src/physics/cam/wv_sat_scream.F90 +++ b/components/eam/src/physics/cam/wv_sat_scream.F90 @@ -14,25 +14,25 @@ module wv_sat_scream use physics_utils, only: rtype use micro_p3_utils, only: T_zerodegc #ifdef SCREAM_CONFIG_IS_CMAKE - use physics_share_f2c, only: cxx_pow, cxx_sqrt, cxx_cbrt, cxx_gamma, cxx_log, & - cxx_log10, cxx_exp, cxx_tanh + use physics_share_f2c, only: scream_pow, scream_sqrt, scream_cbrt, scream_gamma, scream_log, & + scream_log10, scream_exp, scream_tanh #endif implicit none private - public:: qv_sat, MurphyKoop_svp + public:: qv_sat_dry, qv_sat_wet, qv_sat, MurphyKoop_svp contains !=========================================================================================== - real(rtype) function qv_sat(t_atm,p_atm,i_wrt) + real(rtype) function qv_sat_dry(t_atm,p_atm_dry,i_wrt) !------------------------------------------------------------------------------------ - ! Calls polysvp1 to obtain the saturation vapor pressure, and then computes - ! and returns the saturation mixing ratio, with respect to either liquid or ice, + ! Calls MurphyKoop to obtain the saturation vapor pressure, and then computes + ! and returns the dry saturation mixing ratio, with respect to either liquid or ice, ! depending on value of 'i_wrt' !------------------------------------------------------------------------------------ @@ -40,23 +40,79 @@ real(rtype) function qv_sat(t_atm,p_atm,i_wrt) implicit none !Calling parameters: - real(rtype), intent(in) :: t_atm !temperature [K] - real(rtype), intent(in) :: p_atm !pressure [Pa] + real(rtype), intent(in) :: t_atm !temperature [K] + real(rtype), intent(in) :: p_atm_dry !pressure [Pa] + integer, intent(in) :: i_wrt !index, 0 = w.r.t. liquid, 1 = w.r.t. ice + + !Local variables: + real(rtype) :: e_pres !saturation vapor pressure [Pa] + + !e_pres = polysvp1(t_atm,i_wrt) + e_pres = MurphyKoop_svp(t_atm,i_wrt) + qv_sat_dry = ep_2*e_pres/max(1.e-3_rtype,p_atm_dry) + + return + + end function qv_sat_dry + + !=========================================================================================== + real(rtype) function qv_sat(t_atm,p_atm,i_wrt) + + !------------------------------------------------------------------------------------ + ! Legacy for backwards compatibility with eam. Prefer the dry/wet versions going forward. + ! eamxx will use the dry/wet versions. + !------------------------------------------------------------------------------------ + + use micro_p3_utils, only: ep_2 + implicit none + + !Calling parameters: + real(rtype), intent(in) :: t_atm !temperature [K] + real(rtype), intent(in) :: p_atm !pressure [Pa] integer, intent(in) :: i_wrt !index, 0 = w.r.t. liquid, 1 = w.r.t. ice !Local variables: real(rtype) :: e_pres !saturation vapor pressure [Pa] - !e_pres = polysvp1(t_atm,i_wrt) + !e_pres = polysvp1(t_atm,i_wrt) e_pres = MurphyKoop_svp(t_atm,i_wrt) - qv_sat = ep_2*e_pres/max(1.e-3_rtype,(p_atm-e_pres)) + qv_sat = ep_2*e_pres/max(1.e-3_rtype,p_atm-e_pres) return end function qv_sat + !=========================================================================================== - !==========================================================================================! + real(rtype) function qv_sat_wet(t_atm,p_atm_dry,i_wrt,dp_wet,dp_dry) + + !------------------------------------------------------------------------------------ + ! Calls qv_sat_dry to obtain the dry saturation mixing ratio, + ! with respect to either liquid or ice, depending on value of 'i_wrt', + ! and converts it to wet + !------------------------------------------------------------------------------------ + + implicit none + + !Calling parameters: + real(rtype), intent(in) :: t_atm !temperature [K] + real(rtype), intent(in) :: p_atm_dry !pressure [Pa] + real(rtype), intent(in) :: dp_wet !pseudodensity [Pa] + real(rtype), intent(in) :: dp_dry !pseudodensity_dry [Pa] + integer, intent(in) :: i_wrt !index, 0 = w.r.t. liquid, 1 = w.r.t. ice + + !Local variables: + real(rtype) :: qsatdry + + qsatdry = qv_sat_dry(t_atm,p_atm_dry,i_wrt) + qv_sat_wet = qsatdry * dp_dry / dp_wet + + return + + end function qv_sat_wet + !=========================================================================================== + + real(rtype) function MurphyKoop_svp(t, i_type) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index ce614fcd8d1b..87731d56aeee 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit ce614fcd8d1b38e7e638e55e5fcb220531aca178 +Subproject commit 87731d56aeee4bfae4750e17732828ba186367a7 diff --git a/components/eam/src/physics/p3/eam/micro_p3.F90 b/components/eam/src/physics/p3/eam/micro_p3.F90 index 33bc9c32ab94..730736a2dbf2 100644 --- a/components/eam/src/physics/p3/eam/micro_p3.F90 +++ b/components/eam/src/physics/p3/eam/micro_p3.F90 @@ -72,8 +72,8 @@ module micro_p3 ! Bit-for-bit math functions. #ifdef SCREAM_CONFIG_IS_CMAKE - use physics_share_f2c, only: cxx_pow, cxx_sqrt, cxx_cbrt, cxx_gamma, cxx_log, & - cxx_log10, cxx_exp, cxx_expm1, cxx_tanh + use physics_share_f2c, only: scream_pow, scream_sqrt, scream_cbrt, scream_gamma, scream_log, & + scream_log10, scream_exp, scream_expm1, scream_tanh #endif implicit none diff --git a/components/eam/src/physics/p3/scream/micro_p3.F90 b/components/eam/src/physics/p3/scream/micro_p3.F90 index 5723fd172b68..e59bcc08239c 100644 --- a/components/eam/src/physics/p3/scream/micro_p3.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3.F90 @@ -56,12 +56,12 @@ module micro_p3 lookup_table_1a_dum1_c, & p3_qc_autocon_expon, p3_qc_accret_expon - use wv_sat_scream, only:qv_sat + use wv_sat_scream, only:qv_sat_dry ! Bit-for-bit math functions. #ifdef SCREAM_CONFIG_IS_CMAKE - use physics_share_f2c, only: cxx_pow, cxx_sqrt, cxx_cbrt, cxx_gamma, cxx_log, & - cxx_log10, cxx_exp, cxx_expm1, cxx_tanh + use physics_share_f2c, only: scream_pow, scream_sqrt, scream_cbrt, scream_gamma, scream_log, & + scream_log10, scream_exp, scream_expm1, scream_tanh #endif implicit none @@ -384,8 +384,8 @@ SUBROUTINE p3_main_part1(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribe !can be made consistent with E3SM definition of latent heat rho(k) = dpres(k)/dz(k)/g ! pres(k)/(rd*t(k)) inv_rho(k) = 1._rtype/rho(k) - qv_sat_l(k) = qv_sat(t_atm(k),pres(k),0) - qv_sat_i(k) = qv_sat(t_atm(k),pres(k),1) + qv_sat_l(k) = qv_sat_dry(t_atm(k),pres(k),0) + qv_sat_i(k) = qv_sat_dry(t_atm(k),pres(k),1) qv_supersat_i(k) = qv(k)/qv_sat_i(k)-1._rtype @@ -960,7 +960,8 @@ subroutine p3_main_part3(kts, kte, kbot, ktop, kdir, & inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc) + ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, & + diag_eff_radius_qc, diag_eff_radius_qr) implicit none @@ -974,7 +975,8 @@ subroutine p3_main_part3(kts, kte, kbot, ktop, kdir, & qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & mu_c, nu, lamc, mu_r, & lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc + ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, & + diag_eff_radius_qc, diag_eff_radius_qr ! locals integer :: k, dumi, dumii, dumjj, dumzz @@ -1028,6 +1030,7 @@ subroutine p3_main_part3(kts, kte, kbot, ktop, kdir, & ze_rain(k) = nr(k)*(mu_r(k)+6._rtype)*(mu_r(k)+5._rtype)*(mu_r(k)+4._rtype)* & (mu_r(k)+3._rtype)*(mu_r(k)+2._rtype)*(mu_r(k)+1._rtype)/bfb_pow(lamr(k), 6._rtype) ze_rain(k) = max(ze_rain(k),1.e-22_rtype) + diag_eff_radius_qr(k) = 1.5_rtype/lamr(k) else qv(k) = qv(k)+qr(k) th_atm(k) = th_atm(k)-inv_exner(k)*qr(k)*latent_heat_vapor(k)*inv_cp @@ -1122,7 +1125,7 @@ end subroutine p3_main_part3 SUBROUTINE p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,rho_qi,do_predict_nc, do_prescribed_CCN, & + diag_eff_radius_qi,diag_eff_radius_qr,rho_qi,do_predict_nc, do_prescribed_CCN, & dpres,inv_exner,qv2qi_depos_tend,precip_total_tend,nevapr,qr_evap_tend,precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i, & p3_tend_out,mu_c,lamc,liq_ice_exchange,vap_liq_exchange, & vap_ice_exchange,qv_prev,t_prev,col_location & @@ -1173,6 +1176,7 @@ SUBROUTINE p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & real(rtype), intent(out), dimension(its:ite) :: precip_ice_surf ! precipitation rate, solid m s-1 real(rtype), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qc ! effective radius, cloud m real(rtype), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qi ! effective radius, ice m + real(rtype), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qr ! effective radius, rain m real(rtype), intent(out), dimension(its:ite,kts:kte) :: rho_qi ! bulk density of ice kg m-3 real(rtype), intent(out), dimension(its:ite,kts:kte) :: mu_c ! Size distribution shape parameter for radiation real(rtype), intent(out), dimension(its:ite,kts:kte) :: lamc ! Size distribution slope parameter for radiation @@ -1297,6 +1301,7 @@ SUBROUTINE p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & ze_rain = 1.e-22_rtype diag_eff_radius_qc = 10.e-6_rtype ! default value diag_eff_radius_qi = 25.e-6_rtype ! default value + diag_eff_radius_qr = 500.e-6_rtype ! default value diag_vm_qi = 0._rtype diag_diam_qi = 0._rtype rho_qi = 0._rtype @@ -1452,7 +1457,8 @@ SUBROUTINE p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & rho(i,:), inv_rho(i,:), rhofaci(i,:), qv(i,:), th_atm(i,:), qc(i,:), nc(i,:), qr(i,:), nr(i,:), qi(i,:), ni(i,:), & qm(i,:), bm(i,:), latent_heat_vapor(i,:), latent_heat_sublim(i,:), & mu_c(i,:), nu(i,:), lamc(i,:), mu_r(i,:), lamr(i,:), vap_liq_exchange(i,:), & - ze_rain(i,:), ze_ice(i,:), diag_vm_qi(i,:), diag_eff_radius_qi(i,:), diag_diam_qi(i,:), rho_qi(i,:), diag_equiv_reflectivity(i,:), diag_eff_radius_qc(i,:)) + ze_rain(i,:), ze_ice(i,:), diag_vm_qi(i,:), diag_eff_radius_qi(i,:), diag_diam_qi(i,:), rho_qi(i,:), & + diag_equiv_reflectivity(i,:), diag_eff_radius_qc(i,:), diag_eff_radius_qr(i,:)) ! if (debug_ON) call check_values(qv,Ti,it,debug_ABORT,800,col_location) !.............................................. @@ -2233,7 +2239,7 @@ subroutine ice_melting(rho,t_atm,pres,rhofaci, & real(rtype) :: qsat0 if (qi_incld .ge.qsmall .and. t_atm.gt.T_zerodegc) then - qsat0 = qv_sat( T_zerodegc,pres,0 ) + qsat0 = qv_sat_dry( T_zerodegc,pres,0 ) qi2qr_melt_tend = ((table_val_qi2qr_melting+table_val_qi2qr_vent_melt*bfb_cbrt(sc)*bfb_sqrt(rhofaci*rho/mu))*((t_atm- & T_zerodegc)*kap-rho*latent_heat_vapor*dv*(qsat0-qv))*2._rtype*pi/latent_heat_fusion)*ni_incld @@ -2283,7 +2289,7 @@ subroutine ice_cldliq_wet_growth(rho,t_atm,pres,rhofaci, & real(rtype) :: qsat0, dum, dum1 if (qi_incld.ge.qsmall .and. qc_incld+qr_incld.ge.1.e-6_rtype .and. t_atm.lt.T_zerodegc) then - qsat0=qv_sat( T_zerodegc,pres,0 ) + qsat0=qv_sat_dry( T_zerodegc,pres,0 ) qwgrth = ((table_val_qi2qr_melting + table_val_qi2qr_vent_melt*bfb_cbrt(sc)*bfb_sqrt(rhofaci*rho/mu))* & 2._rtype*pi*(rho*latent_heat_vapor*dv*(qsat0-qv)-(t_atm-T_zerodegc)* & @@ -2908,7 +2914,7 @@ subroutine prevent_liq_supersaturation(pres,t_atm,qv,latent_heat_vapor,latent_he - qr2qv_evap_tend*latent_heat_vapor*inv_cp )*dt !qv we would have at end of step if we were saturated with respect to liquid - qsl = qv_sat(T_endstep,pres,0) + qsl = qv_sat_dry(T_endstep,pres,0) ! The balance we seek is: ! qv-qv_sinks*dt+qv_sources*frac*dt=qsl+dqsl_dT*(T correction due to conservation) diff --git a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 index 8e181c21a646..edea27959d25 100644 --- a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 @@ -745,7 +745,6 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) qsmall, & mincld, & inv_cp - use physics_utils, only: calculate_drymmr_from_wetmmr, calculate_wetmmr_from_drymmr !INPUT/OUTPUT VARIABLES type(physics_state), intent(in) :: state @@ -834,7 +833,6 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) real(rtype) :: icimrst(pcols,pver) ! stratus ice mixing ratio - on grid real(rtype) :: icwmrst(pcols,pver) ! stratus water mixing ratio - on grid real(rtype) :: rho(pcols,pver) - real(rtype) :: drout2(pcols,pver) real(rtype) :: reff_rain(pcols,pver) real(rtype) :: col_location(pcols,3),tmp_loc(pcols) ! Array of column lon (index 1) and lat (index 2) integer :: tmpi_loc(pcols) ! Global column index temp array @@ -854,6 +852,9 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) real(rtype) :: icinc(pcols,pver) real(rtype) :: icwnc(pcols,pver) + real(rtype) :: ratio_local(pcols,pver) + real(rtype) :: dtemp(pcols,pver) + integer :: it !timestep counter - integer :: its, ite !horizontal bounds (column start,finish) @@ -865,7 +866,6 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) integer :: icol, ncol, k integer :: psetcols, lchnk integer :: itim_old - real(rtype) :: T_virtual ! For rrtmg optics. specified distribution. real(rtype), parameter :: dcon = 25.e-6_rtype ! Convective size distribution effective radius (um) @@ -984,22 +984,24 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) !------------------------- !Since state constituents from the host model are wet mixing ratios and P3 needs these !constituents in dry mixing ratios, we convert the wet mixing ratios to dry mixing ratio - !while assigning state constituents to the local variables - !NOTE:Function calculate_drymmr_from_wetmmr takes 3 arguments: (number of columns, wet mmr and - ! "wet" water vapor mixing ratio) !--------------------------------------------------------------------------------------- - qv_wet_in = state%q(:,:,1) ! Get "wet" water vapor mixing ratio from state + !Compute dry mixing ratios for all the constituents - qv_dry(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, qv_wet_in, qv_wet_in) - qv_prev_dry(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, qv_prev_wet, qv_wet_in) - cldliq(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, state%q(:,:,ixcldliq), qv_wet_in) - numliq(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, state%q(:,:,ixnumliq), qv_wet_in) - rain(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, state%q(:,:,ixrain), qv_wet_in) - numrain(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, state%q(:,:,ixnumrain), qv_wet_in) - ice(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, state%q(:,:,ixcldice), qv_wet_in) - qm(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, state%q(:,:,ixcldrim), qv_wet_in) !Aaron, changed ixqm to ixcldrim to match Kai's code - numice(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, state%q(:,:,ixnumice), qv_wet_in) - rimvol(:ncol,:pver) = calculate_drymmr_from_wetmmr(ncol, pver, state%q(:,:,ixrimvol), qv_wet_in) + !The conversion is done via calculation drymmr = (wetmmr * wetdp) / drydp + ratio_local(:ncol,:pver) = state%pdel(:ncol,:pver)/state%pdeldry(:ncol,:pver) + + qv_dry (:ncol,:pver) = state%q(:ncol,:pver, 1) *ratio_local(:ncol,:pver) + cldliq (:ncol,:pver) = state%q(:ncol,:pver, ixcldliq) *ratio_local(:ncol,:pver) + numliq (:ncol,:pver) = state%q(:ncol,:pver, ixnumliq) *ratio_local(:ncol,:pver) + rain (:ncol,:pver) = state%q(:ncol,:pver, ixrain) *ratio_local(:ncol,:pver) + numrain (:ncol,:pver) = state%q(:ncol,:pver, ixnumrain) *ratio_local(:ncol,:pver) + ice (:ncol,:pver) = state%q(:ncol,:pver, ixcldice) *ratio_local(:ncol,:pver) + !Aaron, changed ixqm to ixcldrim to match Kai's code + qm (:ncol,:pver) = state%q(:ncol,:pver, ixcldrim) *ratio_local(:ncol,:pver) + numice (:ncol,:pver) = state%q(:ncol,:pver, ixnumice) *ratio_local(:ncol,:pver) + rimvol (:ncol,:pver) = state%q(:ncol,:pver, ixrimvol) *ratio_local(:ncol,:pver) + + qv_prev_dry(:ncol,:pver) = qv_prev_wet(:ncol,:pver) *ratio_local(:ncol,:pver) ! COMPUTE GEOMETRIC THICKNESS OF GRID & CONVERT T TO POTENTIAL TEMPERATURE !============== @@ -1007,15 +1009,16 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) ! used by all parameterizations, such as P3 and SHOC. ! This would take a bit more work, so we have decided to delay this task ! until a later stage of code cleanup. - inv_exner(:ncol,:pver) = 1._rtype/((state%pmiddry(:ncol,:pver)*1.e-5_rtype)**(rair*inv_cp)) + + inv_exner(:ncol,:pver) = 1._rtype/((state%pmid(:ncol,:pver)*1.e-5_rtype)**(rair*inv_cp)) do icol = 1,ncol do k = 1,pver ! Note, there is a state%zi variable that could be used to calculate ! dz, but that is in a wet coordinate frame rather than dry. Now that ! P3 is using dry MMR we instead calculated dz using virtual ! temperature and pressure. - T_virtual = state%t(icol,k) * (1.0 + qv_dry(icol,k)*(1.0*mwdry/mwh2o - 1.0)) - dz(icol,k) = (rair/gravit) * state%pdeldry(icol,k) * T_virtual / state%pmiddry(icol,k) + + dz(icol,k) = state%zi(icol,k) - state%zi(icol,k+1) th(icol,k) = state%t(icol,k)*inv_exner(icol,k) end do end do @@ -1024,8 +1027,10 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) ite = state%ncol kts = 1 kte = pver + +!OG do we want dry or wet pressure here? pres = state%pmiddry(:,:) - ! Initialize the raidation dependent variables. + ! Initialize the radiation dependent variables. mu = 0.0_rtype !mucon lambdac = 0.0_rtype !(mucon + 1._rtype)/dcon dei = 50.0_rtype !deicon @@ -1107,6 +1112,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) kte, & ! IN vertical index upper bound - rel(its:ite,kts:kte), & ! OUT effective radius, cloud m rei(its:ite,kts:kte), & ! OUT effective radius, ice m + reff_rain(its:ite,kts:kte), & ! OUT effective radius, rain m rho_qi(its:ite,kts:kte), & ! OUT bulk density of ice kg m-3 do_predict_nc, & ! IN .true.=prognostic Nc, .false.=specified Nc do_prescribed_CCN, & ! IN @@ -1184,19 +1190,27 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) !================ !Since the host model needs wet mixing ratio tendencies(state vector has wet mixing ratios), !we need to convert dry mixing ratios from P3 to wet mixing ratios before extracting tendencies - !NOTE: water vapor mixing ratio argument in calculate_wetmmr_from_drymmr function has to be dry water vapor mixing ratio - - qv_wet_out(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, qv_dry, qv_dry) - cldliq(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, cldliq, qv_dry) - numliq(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, numliq, qv_dry) - rain(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, rain, qv_dry) - numrain(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, numrain, qv_dry) - ice(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, ice, qv_dry) - numice(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, numice, qv_dry) - qm(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, qm, qv_dry) - rimvol(:ncol,:pver) = calculate_wetmmr_from_drymmr(ncol, pver, rimvol, qv_dry) - - temp(:ncol,:pver) = th(:ncol,:pver)/inv_exner(:ncol,:pver) + + ratio_local(:ncol,:pver) = state%pdeldry(:ncol,:pver)/state%pdel(:ncol,:pver) + + qv_wet_out (:ncol,:pver) = qv_dry (:ncol,:pver) *ratio_local(:ncol,:pver) + cldliq (:ncol,:pver) = cldliq (:ncol,:pver) *ratio_local(:ncol,:pver) + numliq (:ncol,:pver) = numliq (:ncol,:pver) *ratio_local(:ncol,:pver) + rain (:ncol,:pver) = rain (:ncol,:pver) *ratio_local(:ncol,:pver) + numrain (:ncol,:pver) = numrain(:ncol,:pver) *ratio_local(:ncol,:pver) + ice (:ncol,:pver) = ice (:ncol,:pver) *ratio_local(:ncol,:pver) + qm (:ncol,:pver) = qm (:ncol,:pver) *ratio_local(:ncol,:pver) + numice (:ncol,:pver) = numice (:ncol,:pver) *ratio_local(:ncol,:pver) + rimvol (:ncol,:pver) = rimvol (:ncol,:pver) *ratio_local(:ncol,:pver) + + !compute temperature tendency as calculated by P3 + dtemp(:ncol,:pver) = th(:ncol,:pver)/inv_exner(:ncol,:pver) - state%t(:ncol,:pver) + !rescale temperature tendency to conserve entahly: + !physics is supposed to conserve quantity dp*(cpdry*T+Lv*qv+Ll*ql) with dp=wetdp, but + !since P3 is dry, it conserves it for drydp. Scaling of temperature tendencies is required to fix it. + dtemp(:ncol,:pver) = dtemp(:ncol,:pver) *ratio_local(:ncol,:pver) + + temp(:ncol,:pver) = dtemp(:ncol,:pver) + state%t(:ncol,:pver) ptend%s(:ncol,:pver) = cpair*( temp(:ncol,:pver) - state%t(:ncol,:pver) )/dtime ptend%q(:ncol,:pver,1) = ( max(0._rtype,qv_wet_out(:ncol,:pver) ) - state%q(:ncol,:pver,1) )/dtime ptend%q(:ncol,:pver,ixcldliq) = ( max(0._rtype,cldliq(:ncol,:pver) ) - state%q(:ncol,:pver,ixcldliq) )/dtime @@ -1341,23 +1355,15 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) !! !! Rain/Snow effective diameter !! - drout2 = 0._rtype - reff_rain = 0._rtype aqrain = 0._rtype anrain = 0._rtype freqr = 0._rtype ! Prognostic precipitation where (rain(:ncol,top_lev:) >= 1.e-7_rtype) - drout2(:ncol,top_lev:) = avg_diameter( & - rain(:ncol,top_lev:), & - numrain(:ncol,top_lev:) * rho(:ncol,top_lev:), & - rho(:ncol,top_lev:), rho_h2o) - aqrain(:ncol,top_lev:) = rain(:ncol,top_lev:) * cld_frac_r(:ncol,top_lev:) anrain(:ncol,top_lev:) = numrain(:ncol,top_lev:) * cld_frac_r(:ncol,top_lev:) freqr(:ncol,top_lev:) = cld_frac_r(:ncol,top_lev:) - reff_rain(:ncol,top_lev:) = drout2(:ncol,top_lev:) * & - 1.5_rtype * 1.e6_rtype + reff_rain(:ncol,top_lev:) = reff_rain(:ncol,top_lev:) * 1.e6_rtype end where !====================== COSP Specific Outputs START ======================! diff --git a/components/eam/src/physics/p3/scream/micro_p3_utils.F90 b/components/eam/src/physics/p3/scream/micro_p3_utils.F90 index da390e48d324..979cfde9ac8f 100644 --- a/components/eam/src/physics/p3/scream/micro_p3_utils.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3_utils.F90 @@ -109,7 +109,7 @@ subroutine micro_p3_utils_init(cpair,rair,rh2o,rhoh2o,mwh2o,mwdry,gravit,latvap, piov6 = pi*sxth ! maximum total ice concentration (sum of all categories) - max_total_ni = 500.e+3_rtype !(m) + max_total_ni = 740.e+3_rtype !(m) ! droplet concentration (m-3) nccnst = 200.e+6_rtype diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 7839e3a4e033..e2aae3e97b32 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -12,6 +12,10 @@ if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0") cmake_policy(SET CMP0074 NEW) endif() +set (EAMXX_VERSION_MAJOR 1) +set (EAMXX_VERSION_MINOR 0) +set (EAMXX_VERSION_PATCH 0) + if ($ENV{SCREAM_FORCE_CONFIG_FAIL}) message(FATAL_ERROR "Failed, as instructed by environment") endif() @@ -20,12 +24,13 @@ endif() set (EKAT_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat/cmake) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake + ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modules ${EKAT_CMAKE_PATH} - ${EKAT_CMAKE_PATH}/pkg_build + ${EKAT_CMAKE_PATH}/tpls ) if (SCREAM_CIME_BUILD) list(APPEND CMAKE_MODULE_PATH - ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cime) + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cime) endif () if (Kokkos_ENABLE_CUDA) @@ -51,12 +56,8 @@ endif() # to be on. For now, simply ensure Kokkos Serial is enabled option (Kokkos_ENABLE_SERIAL "" ON) -# MAM support requires C++17 -- hopefully SCREAM itself will get there soon -if (SCREAM_ENABLE_MAM) - set(CMAKE_CXX_STANDARD 17) -else() - set(CMAKE_CXX_STANDARD 14) -endif() +# We want to use C++17 in EAMxx +set(CMAKE_CXX_STANDARD 17) if (NOT SCREAM_CIME_BUILD) project(SCREAM CXX C Fortran) @@ -66,18 +67,19 @@ if (NOT SCREAM_CIME_BUILD) list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifport") endif() - # Print the sha of the last commit (useful to double check which version was tested on CDash) - execute_process (COMMAND git rev-parse HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - OUTPUT_VARIABLE LAST_GIT_COMMIT_SHA - OUTPUT_STRIP_TRAILING_WHITESPACE) - set(LAST_GIT_COMMIT_SHA ${LAST_GIT_COMMIT_SHA} CACHE STRING "The sha of the last git commit.") - message(STATUS "The sha of the last commit is ${LAST_GIT_COMMIT_SHA}") else() # Ensure our languages are all enabled enable_language(C CXX Fortran) endif() +# Print the sha of the last commit (useful to double check which version was tested on CDash) +execute_process (COMMAND git rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE LAST_GIT_COMMIT_SHA + OUTPUT_STRIP_TRAILING_WHITESPACE) +set(EAMXX_GIT_VERSION ${LAST_GIT_COMMIT_SHA} CACHE STRING "The sha of the last git commit.") +message(STATUS "The sha of the last commit is ${EAMXX_GIT_VERSION}") + set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)") # Set the scream base and src directory, to be used across subfolders @@ -216,10 +218,8 @@ if (NOT SCREAM_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") endif() -### The following test only runs on quartz or docker container -if (NOT DEFINED RUN_ML_CORRECTION_TEST) - set(RUN_ML_CORRECTION_TEST FALSE) -endif() +# For now, only used in share/grid/remap/refining_remapper_rma.*pp +option (EAMXX_ENABLE_EXPERIMENTAL_CODE "Compile one-sided MPI for refining remappers" OFF) # Handle input root if (SCREAM_MACHINE AND NOT SCREAM_INPUT_ROOT) @@ -272,6 +272,7 @@ endif() set (SCREAM_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/scream CACHE PATH "" FORCE) set (TOPO_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/cam/topo CACHE PATH "" FORCE) +set (IOP_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/cam/scam/iop CACHE PATH "" FORCE) # # Handle test level @@ -472,6 +473,7 @@ if (SCREAM_CIME_BUILD AND SCREAM_DYN_TARGET STREQUAL "theta-l_kokkos") set (DEFAULT_SCREAM_DYNAMICS_DYCORE "Homme") endif() +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" OFF) set(SCREAM_DYNAMICS_DYCORE ${DEFAULT_SCREAM_DYNAMICS_DYCORE} CACHE STRING "The name of the dycore to be used for dynamics. If NONE, then any code/test requiring dynamics is disabled.") @@ -503,8 +505,6 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) - include(BuildCprnc) - BuildCprnc() endif() # Generate scream_config.h and scream_config.f diff --git a/components/eamxx/cime_config/buildlib_cmake b/components/eamxx/cime_config/buildlib_cmake index 155e44efc3c7..bfe56901837c 100755 --- a/components/eamxx/cime_config/buildlib_cmake +++ b/components/eamxx/cime_config/buildlib_cmake @@ -25,11 +25,18 @@ def buildlib(bldroot, installpath, case): expect (len(tokens) % 2 == 0, "Error! SCREAM_CMAKE_OPTIONS should contain a string of the form 'option1 value1 option2 value2 ...'\n") it = iter(tokens) - cmake_args = "" + # Parse all options and put them in a dict first. This allows to overwrite options via + # ./xmlchange --append SCRAM_CMAKE_OPTIONS="NAME VALUE" + # rather than having to reset them all, running ./xmlquery first to see what the others are + cmake_args_dict = {} for item in it: - cmake_args += " -D{}={}".format(item,next(it)) + cmake_args_dict[item] = next(it) + + cmake_args = "" + for k,v in cmake_args_dict.items(): + cmake_args += f" -D{k}={v}" - atm_dyn_tgt = case.get_value("ATM_DYN_TARGET") + atm_dyn_tgt = case.get_value("CAM_TARGET") cmake_args += " -DSCREAM_DYN_TARGET={}".format(atm_dyn_tgt) cmake_args += " -DSCREAM_CIME_BUILD=ON" diff --git a/components/eamxx/cime_config/buildnml b/components/eamxx/cime_config/buildnml index 0f4dceaf0308..9fa6fbbb933c 100755 --- a/components/eamxx/cime_config/buildnml +++ b/components/eamxx/cime_config/buildnml @@ -26,7 +26,7 @@ import os, sys from CIME.case import Case from CIME.utils import expect, safe_copy, SharedArea, run_cmd_no_fail -from CIME.buildlib import parse_input +from CIME.buildnml import parse_input from eamxx_buildnml import create_raw_xml_file, create_input_files, create_input_data_list_file, \ do_cime_vars_on_yaml_output_files diff --git a/components/eamxx/cime_config/config_component.xml b/components/eamxx/cime_config/config_component.xml index 2106d7ca2e5c..0aeccd1e6d24 100644 --- a/components/eamxx/cime_config/config_component.xml +++ b/components/eamxx/cime_config/config_component.xml @@ -13,7 +13,7 @@ Name of atmospheric component - + char theta-l_kokkos theta-l_kokkos @@ -29,6 +29,8 @@ SCREAM_NP 4 SCREAM_NUM_VERTICAL_LEV 72 SCREAM_NUM_TRACERS 10 SCREAM_NP 4 SCREAM_NUM_VERTICAL_LEV 128 SCREAM_NUM_TRACERS 10 + SCREAM_NP 4 SCREAM_NUM_VERTICAL_LEV 72 SCREAM_NUM_TRACERS 10 + SCREAM_NP 4 SCREAM_NUM_VERTICAL_LEV 128 SCREAM_NUM_TRACERS 10 build_component_scream env_build.xml diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 1e7626d46f47..db05bd2e20e1 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -8,24 +8,27 @@ from collections import OrderedDict import xml.etree.ElementTree as ET - -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","cime") -sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) +import xml.dom.minidom as md # Add path to scream libs sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts")) -# Cime imports -from standard_script_setup import * # pylint: disable=wildcard-import -from CIME.utils import expect, safe_copy, SharedArea, run_cmd_no_fail - # SCREAM imports from eamxx_buildnml_impl import get_valid_selectors, get_child, refine_type, \ - resolve_all_inheritances, gen_atm_proc_group, check_all_values + resolve_all_inheritances, gen_atm_proc_group, check_all_values, find_node +from atm_manip import apply_atm_procs_list_changes_from_buffer, apply_non_atm_procs_list_changes_from_buffer -from utils import ensure_yaml +from utils import ensure_yaml # pylint: disable=no-name-in-module ensure_yaml() import yaml +from yaml_utils import Bools,Ints,Floats,Strings,array_representer + +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","cime") +sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) + +# Cime imports +from standard_script_setup import * # pylint: disable=wildcard-import +from CIME.utils import expect, safe_copy, SharedArea logger = logging.getLogger(__name__) # pylint: disable=undefined-variable @@ -41,7 +44,7 @@ # Examples: # - constraints="ge 0; lt 4" means the value V must satisfy V>=0 && V<4. # - constraints="mod 2 eq 0" means the value V must be a multiple of 2. -METADATA_ATTRIBS = ("type", "valid_values", "locked", "constraints", "inherit") +METADATA_ATTRIBS = ("type", "valid_values", "locked", "constraints", "inherit", "doc", "append") ############################################################################### def do_cime_vars(entry, case, refine=False, extra=None): @@ -101,6 +104,107 @@ def do_cime_vars(entry, case, refine=False, extra=None): return entry +############################################################################### +def perform_consistency_checks(case, xml): +############################################################################### + """ + There may be separate parts of the xml that must satisfy some consistency + Here, we run any such check, so we can catch errors before submit time + + >>> from eamxx_buildnml_impl import MockCase + >>> xml_str = ''' + ... + ... + ... 3 + ... + ... + ... ''' + >>> import xml.etree.ElementTree as ET + >>> xml = ET.fromstring(xml_str) + >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':24, 'REST_OPTION':'nsteps'}) + >>> perform_consistency_checks(case,xml) + >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':2, 'REST_OPTION':'nsteps'}) + >>> perform_consistency_checks(case,xml) + Traceback (most recent call last): + CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. + Please, ensure restart happens on a step when rad is ON + >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':10800, 'REST_OPTION':'nseconds'}) + >>> perform_consistency_checks(case,xml) + >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':7200, 'REST_OPTION':'nseconds'}) + >>> perform_consistency_checks(case,xml) + Traceback (most recent call last): + CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. + Please, ensure restart happens on a step when rad is ON + rest_tstep: 7200 + rad_testep: 10800.0 + >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':180, 'REST_OPTION':'nminutes'}) + >>> perform_consistency_checks(case,xml) + >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':120, 'REST_OPTION':'nminutes'}) + >>> perform_consistency_checks(case,xml) + Traceback (most recent call last): + CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. + Please, ensure restart happens on a step when rad is ON + rest_tstep: 7200 + rad_testep: 10800.0 + >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':6, 'REST_OPTION':'nhours'}) + >>> perform_consistency_checks(case,xml) + >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':8, 'REST_OPTION':'nhours'}) + >>> perform_consistency_checks(case,xml) + Traceback (most recent call last): + CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. + Please, ensure restart happens on a step when rad is ON + rest_tstep: 28800 + rad_testep: 10800.0 + >>> case = MockCase({'ATM_NCPL':'12', 'REST_N':2, 'REST_OPTION':'ndays'}) + >>> perform_consistency_checks(case,xml) + >>> case = MockCase({'ATM_NCPL':'10', 'REST_N':2, 'REST_OPTION':'ndays'}) + >>> perform_consistency_checks(case,xml) + Traceback (most recent call last): + CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. + Please, ensure restart happens on a step when rad is ON + For daily (or less frequent) restart, rad_frequency must divide ATM_NCPL + """ + + # RRTMGP can be supercycled. Restarts cannot fall in the middle + # of a rad superstep + rrtmgp = find_node(xml,"rrtmgp") + rest_opt = case.get_value("REST_OPTION") + if rrtmgp is not None and rest_opt is not None and rest_opt not in ["never","none"]: + rest_n = int(case.get_value("REST_N")) + rad_freq = int(find_node(rrtmgp,"rad_frequency").text) + atm_ncpl = int(case.get_value("ATM_NCPL")) + atm_tstep = 86400 / atm_ncpl + rad_tstep = atm_tstep * rad_freq + + + if rad_freq==1: + pass + elif rest_opt in ["nsteps", "nstep"]: + expect (rest_n % rad_freq == 0, + "rrtmgp::rad_frequency incompatible with restart frequency.\n" + " Please, ensure restart happens on a step when rad is ON") + elif rest_opt in ["nseconds", "nsecond", "nminutes", "nminute", "nhours", "nhour"]: + if rest_opt in ["nseconds", "nsecond"]: + factor = 1 + elif rest_opt in ["nminutes", "nminute"]: + factor = 60 + else: + factor = 3600 + + rest_tstep = factor*rest_n + expect (rest_tstep % rad_tstep == 0, + "rrtmgp::rad_frequency incompatible with restart frequency.\n" + " Please, ensure restart happens on a step when rad is ON\n" + f" rest_tstep: {rest_tstep}\n" + f" rad_testep: {rad_tstep}") + + else: + # for "very infrequent" restarts, we request rad_freq to divide atm_ncpl + expect (atm_ncpl % rad_freq ==0, + "rrtmgp::rad_frequency incompatible with restart frequency.\n" + " Please, ensure restart happens on a step when rad is ON\n" + " For daily (or less frequent) restart, rad_frequency must divide ATM_NCPL") + ############################################################################### def ordered_dump(data, item, Dumper=yaml.SafeDumper, **kwds): ############################################################################### @@ -108,6 +212,7 @@ def ordered_dump(data, item, Dumper=yaml.SafeDumper, **kwds): Copied from: https://stackoverflow.com/a/21912744 Added ability to pass filename """ + class OrderedDumper(Dumper): pass def _dict_representer(dumper, data): @@ -116,6 +221,12 @@ def _dict_representer(dumper, data): data.items()) OrderedDumper.add_representer(OrderedDict, _dict_representer) + # These allow to dump arrays with a tag specifying the type + OrderedDumper.add_representer(Bools, array_representer) + OrderedDumper.add_representer(Ints, array_representer) + OrderedDumper.add_representer(Floats, array_representer) + OrderedDumper.add_representer(Strings, array_representer) + if isinstance(item, str) and item.endswith(".yaml"): # Item is a filepath with open(item, "w") as fd: @@ -233,27 +344,50 @@ def evaluate_selectors(element, case, ez_selectors): CIME.utils.CIMEError: ERROR: child 'var1' element without selectors occurred after other parameter elements for this parameter """ - child_values = {} # elem_name -> evaluated XML element + selected_child = {} # elem_name -> evaluated XML element children_to_remove = [] + child_base_value = {} # map elme name to values to be appended to if append=="base" + child_type = {} # map elme name to its type (since only first entry may have type specified) for child in element: - child_name = child.tag - child_val = child.text - # Note: in our system, an XML element is either a "node" (has children) # or a "leaf" (has a value). has_children = len(child) > 0 if has_children: evaluate_selectors(child, case, ez_selectors) else: + child_name = child.tag + child.text = None if child.text is None else child.text.strip(' \n') + child_val = child.text selectors = child.attrib + + if child_name not in child_type: + child_type[child_name] = selectors["type"] if "type" in selectors.keys() else "unset" + + is_array = child_type[child_name].startswith("array") + expect (is_array or "append" not in selectors.keys(), + "The 'append' metadata attribute is only supported for entries of array type\n" + f" param name: {child_name}\n" + f" param type: {child_type[child_name]}") + + append = selectors["append"] if "append" in selectors.keys() else "no" + expect (append in ["no","base","last"], + "Unrecognized value for 'append' attribute\n" + + f" param name : {child_name}\n" + + f" append value: {append}\n" + + " valid values: base, last\n") if selectors: all_match = True - is_first = False + had_case_selectors = False for k, v in selectors.items(): # Metadata attributes are used only when it's time to generate the input files if k in METADATA_ATTRIBS: + if k=="type" and child_name in selected_child.keys(): + if "type" in selected_child[child_name].attrib: + expect (v==selected_child[child_name].attrib["type"], + f"The 'type' attribute of {child_name} is not consistent across different selectors") continue + had_case_selectors = True val_re = re.compile(v) if k in ez_selectors: @@ -279,39 +413,57 @@ def evaluate_selectors(element, case, ez_selectors): expect(val is not None, "Bad selector '{0}' for child '{1}'. '{0}' is not a valid case value or easy selector".format(k, child_name)) - if val is None or val_re.match(val) is None: all_match = False + children_to_remove.append(child) break if all_match: - if child_name in child_values: - orig_child = child_values[child_name] - orig_child.text = do_cime_vars(child_val, case) + if child_name in selected_child.keys(): + orig_child = selected_child[child_name] + if append=="base": + orig_child.text = child_base_value[child_name] + "," + child.text + elif append=="last": + orig_child.text = orig_child.text + "," + child.text + else: + orig_child.text = child.text + children_to_remove.append(child) else: - is_first = True - child_values[child_name] = child - child.text = do_cime_vars(child_val, case) + # If all selectors were the METADATA_ATTRIB ones, then this is the "base" value + if not had_case_selectors: + child_base_value[child_name] = child.text + selected_child[child_name] = child # Make a copy of selectors.keys(), since selectors=child.attrib, # and we might delete an entry, causing the error # RuntimeError: dictionary changed size during iteration - for k in list(selectors.keys()): - if k not in METADATA_ATTRIBS: - del child.attrib[k] - - if not is_first: - children_to_remove.append(child) else: - expect(child_name not in child_values, + expect(child_name not in selected_child, "child '{}' element without selectors occurred after other parameter elements for this parameter".format(child_name)) - child_values[child_name] = child + child_base_value[child_name] = child.text + selected_child[child_name] = child child.text = do_cime_vars(child_val, case) for child_to_remove in children_to_remove: element.remove(child_to_remove) +############################################################################### +def expand_cime_vars(element, case): +############################################################################### + """ + Expand all CIME variables inside an XML node text + """ + + for child in element: + # Note: in our system, an XML element is either a "node" (has children) + # or a "leaf" (has a value). + has_children = len(child) > 0 + if has_children: + expand_cime_vars(child, case) + else: + child.text = do_cime_vars(child.text, case) + ############################################################################### def _create_raw_xml_file_impl(case, xml): ############################################################################### @@ -327,7 +479,7 @@ def _create_raw_xml_file_impl(case, xml): ... ... ... - ... (P1,P2) + ... P1,P2 ... ... zero ... @@ -350,7 +502,7 @@ def _create_raw_xml_file_impl(case, xml): >>> import pprint >>> pp = pprint.PrettyPrinter(indent=4) >>> pp.pprint(d) - OrderedDict([ ('atm_procs_list', '(P1,P2)'), + OrderedDict([ ('atm_procs_list', 'P1,P2'), ('prop2', 'one'), ('prop1', 'zero'), ('P1', OrderedDict([('prop1', 'two')])), @@ -364,7 +516,7 @@ def _create_raw_xml_file_impl(case, xml): ... ... ... - ... (P1,P2) + ... P1,P2 ... ... zero ... @@ -388,7 +540,7 @@ def _create_raw_xml_file_impl(case, xml): >>> import pprint >>> pp = pprint.PrettyPrinter(indent=4) >>> pp.pprint(d) - OrderedDict([ ('atm_procs_list', '(P1,P2)'), + OrderedDict([ ('atm_procs_list', 'P1,P2'), ('prop2', 'one'), ('prop1', 'zero'), ('P1', OrderedDict([('prop1', 'two_selected')])), @@ -402,7 +554,7 @@ def _create_raw_xml_file_impl(case, xml): ... ... ... - ... (P1,P2) + ... P1,P2 ... ... 1 ... true @@ -432,7 +584,7 @@ def _create_raw_xml_file_impl(case, xml): >>> import pprint >>> pp = pprint.PrettyPrinter(indent=4) >>> pp.pprint(d) - OrderedDict([ ('atm_procs_list', '(P1,P2)'), + OrderedDict([ ('atm_procs_list', 'P1,P2'), ('prop2', 'one'), ('number_of_subcycles', 1), ('enable_precondition_checks', True), @@ -457,55 +609,74 @@ def _create_raw_xml_file_impl(case, xml): get_child(xml,"generated_files",remove=True) selectors = get_valid_selectors(xml) - # 1. Resolve all inheritances, and evaluate all selectors + # 1. Evaluate all selectors evaluate_selectors(xml, case, selectors) + + # 2. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that may alter + # which atm processes are used + apply_atm_procs_list_changes_from_buffer (case,xml) + + # 3. Resolve all inheritances resolve_all_inheritances(xml) - # 2. Grab the atmosphere_processes macro list, with all the defaults + # 4. Expand any CIME var that appears inside XML nodes text + expand_cime_vars(xml,case) + + # 5. Grab the atmosphere_processes macro list, with all the defaults atm_procs_defaults = get_child(xml,"atmosphere_processes_defaults",remove=True) - # 3. Get atm procs list + # 6. Get atm procs list atm_procs_list = get_child(atm_procs_defaults,"atm_procs_list",remove=True) - # 4. Form the nested list of atm procs needed, append to atmosphere_driver section - atm_procs = gen_atm_proc_group (atm_procs_list.text, atm_procs_defaults) + # 7. Form the nested list of atm procs needed, append to atmosphere_driver section + atm_procs = gen_atm_proc_group(atm_procs_list.text, atm_procs_defaults) atm_procs.tag = "atmosphere_processes" xml.append(atm_procs) + # 8. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that do not alter + # which atm processes are used + apply_non_atm_procs_list_changes_from_buffer (case,xml) + + perform_consistency_checks (case, xml) + return xml ############################################################################### def create_raw_xml_file(case, caseroot): ############################################################################### """ - Create the raw $case/namelist_scream.xml file. This file is intended to be - modified by users via the atmchange script if they want + Create the $case/namelist_scream.xml file. This file is intended to be + modified by users via the atmchange script if they want, to make tweaks to input files (yaml and/or nml). + Note: users calls to atmchange do two things: 1) they add the change + to the SCREAM_ATMCHANGE_BUFFER case variable, and 2) they + call this function, which regenerates the scream xml file from + the defaults, applyin all buffered changes. """ - src = os.path.join(case.get_value("SRCROOT"), "components/eamxx/cime_config/namelist_defaults_scream.xml") - raw_xml_file = os.path.join(caseroot, "namelist_scream.xml") - with open(src, "r") as fd: - defaults = ET.parse(fd) - raw_xml = _create_raw_xml_file_impl(case, defaults.getroot()) - if os.path.exists(raw_xml_file) and case.get_value("SCREAM_HACK_XML"): print("{} already exists and SCREAM_HACK_XML is on, will not overwrite. Remove to regenerate".format(raw_xml_file)) else: - if os.path.exists(raw_xml_file): - print("Regenerating {}. Manual edits will be lost.".format(raw_xml_file)) + print("Regenerating {}. Manual edits will be lost.".format(raw_xml_file)) + + src = os.path.join(case.get_value("SRCROOT"), "components/eamxx/cime_config/namelist_defaults_scream.xml") + + # Some atmchanges will require structural changes to the XML file and must + # be processed early by treating them as if they were made to the defaults file. + with open(src, "r") as fd: + defaults = ET.parse(fd).getroot() + raw_xml = _create_raw_xml_file_impl(case, defaults) check_all_values(raw_xml) with open(raw_xml_file, "w") as fd: - ET.ElementTree(raw_xml).write(fd, method='xml', encoding="unicode") - - # Now that we have our namelist_scream.xml file, we can apply buffered - # atmchange requests. - atmchg_buffer = case.get_value("SCREAM_ATMCHANGE_BUFFER") - if atmchg_buffer: - run_cmd_no_fail("{}/atmchange {} --no-buffer".format(caseroot, atmchg_buffer)) + # dom has better pretty printing than ET in older python versions < 3.9 + dom = md.parseString(ET.tostring(raw_xml, encoding="unicode")) + pretty_xml = dom.toprettyxml(indent=" ") + pretty_xml = os.linesep.join([s for s in pretty_xml.splitlines() + if s.strip()]) + fd.write(pretty_xml) ############################################################################### def convert_to_dict(element): @@ -516,8 +687,8 @@ def convert_to_dict(element): ... ... 1 ... - ... 2,3 - ... two,three + ... 2,3 + ... two,three ... ... ... ''' @@ -535,14 +706,14 @@ def convert_to_dict(element): result = OrderedDict() for child in element: child_name = child.tag.replace("__", " ") - child_val = child.text has_children = len(child) > 0 - if not has_children: + if has_children: + result[child_name] = convert_to_dict(child) + else: + child_val = child.text force_type = None if "type" not in child.attrib.keys() else child.attrib["type"] result[child_name] = refine_type(child_val,force_type=force_type) - else: - result[child_name] = convert_to_dict(child) return result @@ -732,8 +903,10 @@ def create_input_data_list_file(caseroot): fd.write("scream_dl_input_{} = {}\n".format(idx, file_path)) ############################################################################### -def do_cime_vars_on_yaml_output_files(case,caseroot): +def do_cime_vars_on_yaml_output_files(case, caseroot): ############################################################################### + from yaml_utils import array_constructor + rundir = case.get_value("RUNDIR") eamxx_xml_file = os.path.join(caseroot, "namelist_scream.xml") @@ -744,11 +917,18 @@ def do_cime_vars_on_yaml_output_files(case,caseroot): out_files_xml = get_child(scorpio,"output_yaml_files",must_exist=False) out_files = out_files_xml.text.split(",") if (out_files_xml is not None and out_files_xml.text is not None) else [] + # Add array parsing knowledge to yaml loader + loader = yaml.SafeLoader + loader.add_constructor("!bools",array_constructor) + loader.add_constructor("!ints",array_constructor) + loader.add_constructor("!floats",array_constructor) + loader.add_constructor("!strings",array_constructor) + # We will also change the 'output_yaml_files' entry in scream_input.yaml, # to point to the copied files in $rundir/data output_yaml_files = [] scream_input_file = os.path.join(rundir,'data','scream_input.yaml') - scream_input = yaml.safe_load(open(scream_input_file,"r")) + scream_input = yaml.load(open(scream_input_file,"r"),Loader=loader) # Determine the physics grid type for use in CIME-var substitution. pgt = 'GLL' @@ -768,7 +948,7 @@ def do_cime_vars_on_yaml_output_files(case,caseroot): safe_copy(src_yaml,dst_yaml) # Now load dst file, and process any CIME var present (if any) - content = yaml.safe_load(open(dst_yaml,"r")) + content = yaml.load(open(dst_yaml,"r"),Loader=loader) do_cime_vars(content,case,refine=True, extra={'PHYSICS_GRID_TYPE': pgt}) @@ -778,7 +958,7 @@ def do_cime_vars_on_yaml_output_files(case,caseroot): # Hence, change default output settings to perform a single AVERAGE step at the end of the run if case.get_value("TESTCASE") in ["ERP", "ERS"]: test_env = case.get_env('test') - stop_n = test_env.get_value("STOP_N") + stop_n = int(test_env.get_value("STOP_N")) stop_opt = test_env.get_value("STOP_OPTION") content['output_control']['Frequency'] = stop_n content['output_control']['frequency_units'] = stop_opt @@ -791,7 +971,6 @@ def do_cime_vars_on_yaml_output_files(case,caseroot): # Now update the output yaml files entry, and dump the new content # of the scream input to YAML file - print ("out list: {}".format(",".join(output_yaml_files))) scream_input["Scorpio"]["output_yaml_files"] = refine_type(",".join(output_yaml_files),"array(string)") with open(scream_input_file, "w") as fd: fd.write( diff --git a/components/eamxx/cime_config/eamxx_buildnml_impl.py b/components/eamxx/cime_config/eamxx_buildnml_impl.py index aaf7fe9d9f17..2fa00b7f4a4b 100644 --- a/components/eamxx/cime_config/eamxx_buildnml_impl.py +++ b/components/eamxx/cime_config/eamxx_buildnml_impl.py @@ -5,6 +5,8 @@ sys.path.append(_CIMEROOT) from CIME.utils import expect +from yaml_utils import make_array + ############################################################################### class MockCase(object): @@ -23,58 +25,7 @@ def get_value(self, key): return None ############################################################################### -def parse_string_as_list (string): -############################################################################### - """ - Takes a string representation of nested list and creates - a nested list of stirng. For instance, with - s = "(a,b,(c,d),e) - l = parse_string_as_list - we would have l = ['a', 'b', '(c,d)', 'e'] - - >>> s = '(a,(b,c))' - >>> l = parse_string_as_list(s) - >>> len(l) - 2 - >>> l[0] == 'a' - True - >>> l[1] == '(b,c)' - True - >>> ###### NOT STARTING/ENDING WITH PARENTHESES ####### - >>> s = '(a,b,' - >>> l = parse_string_as_list(s) - Traceback (most recent call last): - ValueError: Input string must start with '(' and end with ')'. - >>> ################ UNMATCHED PARENTHESES ############## - >>> s = '(a,(b)' - >>> l = parse_string_as_list(s) - Traceback (most recent call last): - ValueError: Unmatched parentheses in input string - """ - - if string[0]!='(' or string[-1]!=')': - raise ValueError ("Input string must start with '(' and end with ')'.") - - sub_open = string.find('(',1) - sub_close = string.rfind(')',0,-1) - if not (sub_open>=0)==(sub_close>=0): - raise ValueError ("Unmatched parentheses in input string") - - # Prevent empty string to pollute s.split() - my_split = lambda str : [s for s in str.split(',') if s.strip() != ''] - - if sub_open>=0: - l = [] - l.extend(my_split(string[1:sub_open-1])) - l.append(string[sub_open:sub_close+1]) - l.extend(my_split(string[sub_close+2:-1])) - else: - l = my_split(string[1:-1]) - - return l - -############################################################################### -def is_array_type (name): +def is_array_type(name): ############################################################################### """ >>> is_array_type('array(T)') @@ -84,10 +35,10 @@ def is_array_type (name): >>> is_array_type('array(T)') True """ - return name[0:6]=="array(" and name[-1]==")" + return name is not None and name[0:6]=="array(" and name[-1]==")" ############################################################################### -def array_elem_type (name): +def array_elem_type(name): ############################################################################### """ >>> print(array_elem_type('array(T)')) @@ -200,94 +151,78 @@ def refine_type(entry, force_type=None): >>> refine_type(e)==e True >>> e = 'a,b' - >>> refine_type(e)==['a','b'] + >>> refine_type(e,'array(string)')==['a','b'] True >>> e = 'true,falsE' - >>> refine_type(e)==[True,False] + >>> refine_type(e,'array(logical)')==[True,False] True >>> e = '1' >>> refine_type(e,force_type='real')==1.0 True - >>> e = '1,b' - >>> refine_type(e)==[1,'b',True] - Traceback (most recent call last): - CIME.utils.CIMEError: ERROR: List '1,b' has inconsistent types inside >>> e = '1.0' >>> refine_type(e,force_type='my_type') Traceback (most recent call last): - NameError: Bad force_type: my_type + NameError: ERROR: Invalid/unsupported force type 'my_type' >>> e = 'true,falsE' >>> refine_type(e,'logical') Traceback (most recent call last): - CIME.utils.CIMEError: ERROR: Error! Invalid type 'logical' for an array. + ValueError: Could not refine 'true,falsE' as type 'logical' >>> refine_type(e,'array(logical)') [True, False] >>> refine_type('', 'array(string)') [] - >>> refine_type('', 'array(float)') + >>> refine_type('', 'array(real)') [] - >>> refine_type(None, 'array(float)') + >>> refine_type(None, 'array(real)') [] """ - # We want to preserve strings representing lists - - if entry: - if (entry[0]=="(" and entry[-1]==")") or \ - (entry[0]=="[" and entry[-1]=="]") : - expect (force_type is None or force_type == "string", - "Error! Invalid force type '{}' for a string representing a list" - .format(force_type)) - return entry - - if "," in entry: - expect (force_type is None or is_array_type(force_type), - "Error! Invalid type '{}' for an array.".format(force_type)) - - elem_type = force_type if force_type is None else array_elem_type(force_type) - result = [refine_type(item.strip(), force_type=elem_type) for item in entry.split(",") if item.strip() != ""] - expected_type = type(result[0]) - for item in result[1:]: - expect(isinstance(item, expected_type), - "List '{}' has inconsistent types inside".format(entry)) - - return result - - elif force_type is not None and is_array_type(force_type): - - return [] + # If force type is unspecified, try to deduce it + if force_type is None: + expect (entry is not None, + "If an entry is None, you must specify the force_type") + else: + elem_valid = ["logical","integer","real","string","file"] + valid = elem_valid + ["array("+e+")" for e in elem_valid] + expect (force_type in valid, exc_type=NameError, + error_msg=f"Invalid/unsupported force type '{force_type}'") + + if is_array_type(force_type): + elem_type = array_elem_type(force_type) + if entry: + try: + result = [refine_type(item.strip(), force_type=elem_type) for item in entry.split(",") if item.strip() != ""] + except ValueError: + expect(False, "List '{entry}' has items not compatible with requested element type '{elem_type}'") + else: + result = [] - if force_type: - try: - elem_type = force_type if not is_array_type(force_type) else array_elem_type(force_type) + return make_array(result, elem_type) - if elem_type == "logical": - if entry.upper() == "TRUE": - elem = True - elif entry.upper() == "FALSE": - elem = False - else: - elem = bool(int(entry)) - - elif elem_type == "integer": - tmp = float(entry) - expect (float(int(tmp))==tmp, "Cannot interpret {} as int".format(entry), exc_type=ValueError) - elem = int(tmp) - elif elem_type == "real": - elem = float(entry) - elif elem_type in ["string", "file"]: - elem = str(entry) + # Not an array (or no force type passed) + elem_type = force_type + try: + if elem_type == "logical": + if entry.upper() == "TRUE": + return True + elif entry.upper() == "FALSE": + return False else: - raise NameError ("Bad force_type: {}".format(force_type)) + return bool(int(entry)) - if is_array_type(force_type): - return [elem] - else: - return elem + elif elem_type == "integer": + tmp = float(entry) + expect (float(int(tmp))==tmp, f"Cannot interpret {entry} as int", exc_type=ValueError) + return int(tmp) + elif elem_type == "real": + return float(entry) + elif elem_type in ["string", "file"]: + return str(entry) - except ValueError as e: - raise ValueError ("Could not use '{}' as type '{}'".format(entry, force_type)) from e + except ValueError as e: + raise ValueError (f"Could not refine '{entry}' as type '{force_type}'") from e + # No force type provided. Try to infer from value if entry.upper() == "TRUE": return True elif entry.upper() == "FALSE": @@ -303,6 +238,7 @@ def refine_type(entry, force_type=None): v = float(entry) return v except ValueError: + # We ran out of options. Simply return the entry itself return entry ############################################################################### @@ -317,9 +253,9 @@ def derive_type(entry): >>> derive_type('one') 'string' >>> derive_type('one,two') - 'array(string)' - >>> derive_type('true,FALSE') - 'array(logical)' + 'string' + >>> derive_type('truE') + 'logical' """ refined_value = refine_type(entry) @@ -357,7 +293,7 @@ def check_value(elem, value): >>> root = ET.fromstring(xml) >>> check_value(root,'1.5') Traceback (most recent call last): - ValueError: Could not use '1.5' as type 'integer' + ValueError: Could not refine '1.5' as type 'integer' >>> check_value(root,'3') Traceback (most recent call last): CIME.utils.CIMEError: ERROR: Invalid value '3' for element 'a'. Value not in the valid list ('[1, 2]') @@ -510,7 +446,7 @@ def check_all_values(root): check_value(root,root.text) ############################################################################### -def resolve_inheritance (root,elem): +def resolve_inheritance(root, elem): ############################################################################### """ If elem inherits from another node within $root, this function adds all @@ -556,17 +492,26 @@ def resolve_inheritance (root,elem): if not has_child(elem,entry.tag): new_entry = copy.deepcopy(entry) elem.append(new_entry) + else: + # Parent may define the type and/or doc of an entry. We cannot change this + for att in ["type","doc"]: + if att in entry.attrib.keys(): + parent_type = entry.attrib[att] + for child in elem: + if child.tag==entry.tag: + expect (att not in child.attrib.keys(), + f"Do not set '{att}' attribute when parent node already specifies it.") + child.attrib[att] = parent_type for child in elem: resolve_inheritance(root,child) ############################################################################### -def resolve_all_inheritances (root): +def resolve_all_inheritances(root): ############################################################################### """ Resolve all inheritances in the root tree """ - for elem in root: resolve_inheritance(root,elem) @@ -623,38 +568,32 @@ def get_valid_selectors(xml_root): return selectors ############################################################################### -def gen_group_processes (ap_names_str, atm_procs_defaults): +def gen_group_processes(ap_names_str, atm_procs_defaults): ############################################################################### """ - Given a (possibly nested) string representation of an atm group, + Given a comma-separated list of atm procs names, generates the corresponding atm processes as XML nodes. """ group = ET.Element("__APG__") - ap_names_list = parse_string_as_list(ap_names_str) - for ap in ap_names_list: - # The current ap can be itself a group if either: - # - ap = "(ap1,ap2,...,apXYZ)", with each ap possibly itself a group string. - # This group is built on the fly based on the building blocks specs. - # - ap is declared in the XML defaults as an atm proc group (which must store - # the 'atm_procs_list' child, with the string representation of the group. - - if ap[0]=='(': - # Create the atm proc group - proc = gen_atm_proc_group(ap,atm_procs_defaults) - else: - # Get defaults - proc = copy.deepcopy(get_child(atm_procs_defaults,ap)) - - # Check if this pre-defined proc is itself a group, and, if so, - # build all its sub-processes - ptype = get_child(proc,"Type",must_exist=False) - if ptype is not None and ptype.text=="Group": - # This entry of the group is itself a group, with pre-defined - # defaults. Let's add its entries to it - sub_group_procs = get_child(proc,"atm_procs_list").text - proc.extend(gen_group_processes(sub_group_procs,atm_procs_defaults)) + ap_list = [] if ap_names_str is None or ap_names_str=="" else ap_names_str.split(',') + for ap in ap_list: + # The current ap can be itself a group if ap is declared in the XML defaults + # as an atm proc group (which must store the 'atm_procs_list' child, + # with the string representation of the group. + + # Get defaults + proc = copy.deepcopy(get_child(atm_procs_defaults,ap)) + + # Check if this pre-defined proc is itself a group, and, if so, + # build all its sub-processes + ptype = get_child(proc, "Type", must_exist=False) + if ptype is not None and ptype.text=="Group": + # This entry of the group is itself a group, with pre-defined + # defaults. Let's add its entries to it + sub_group_procs = get_child(proc, "atm_procs_list").text + proc.extend(gen_group_processes(sub_group_procs, atm_procs_defaults)) # Append subproc to group group.append(proc) @@ -673,7 +612,7 @@ def gen_atm_proc_group(atm_procs_list, atm_procs_defaults): ... ... ... 1 - ... THE_LIST + ... THE_LIST ... ... ... @@ -682,19 +621,17 @@ def gen_atm_proc_group(atm_procs_list, atm_procs_defaults): ... 3 ... ... - ... (p1,ap2) + ... p1,ap2 ... ... ... ''' >>> import xml.etree.ElementTree as ET >>> defaults = ET.fromstring(xml) - >>> ap_list = '(ap1,(ap2,ap1))' + >>> ap_list = 'ap1,ap2,ap1' >>> apg = gen_atm_proc_group(ap_list,defaults) >>> get_child(apg,'atm_procs_list').text==ap_list True >>> - >>> has_child(apg,'group.ap2_ap1.') - True >>> get_child(apg,'prop1').text=="1" True """ @@ -702,18 +639,17 @@ def gen_atm_proc_group(atm_procs_list, atm_procs_defaults): # Set defaults from atm_proc_group group = ET.Element("__APG__") group.attrib["inherit"] = "atm_proc_group" - resolve_inheritance(atm_procs_defaults,group) + resolve_inheritance(atm_procs_defaults, group) get_child(group,"atm_procs_list").text = atm_procs_list # Create processes - group_procs = gen_group_processes (atm_procs_list, atm_procs_defaults) + group_procs = gen_group_processes(atm_procs_list, atm_procs_defaults) - # Append procs and generate name for the group. - # NOTE: the name of a 'generic' group is 'group.AP1_AP2_..._APN.' - names = [] + # Append procs for c in group_procs: - names.append(c.tag) group.append(c) - group.tag = "group." + '_'.join(names) + '.' + + # Will be set from outside + group.tag = "MISSING" return group diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index df2007ca9941..71c1bfd4f828 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -35,7 +35,7 @@ be lost if SCREAM_HACK_XML is not enabled. --> - + @@ -141,41 +141,51 @@ be lost if SCREAM_HACK_XML is not enabled. so it must be of the form (a,b,...). NOTE: *CANNOT* be changed. --> - - (sc_import,homme,physics,sc_export) + sc_import,homme,physics,sc_export - 1 + 1 true true trace - NONE + + 0 + - ERROR_NO_ATM_PROCS + Group - Sequential + Sequential - NONE - 0 + + + + + + + + - Dynamics moist - 0 + in-fields. <= 0 disables hashing. --> + 18 @@ -185,6 +195,7 @@ be lost if SCREAM_HACK_XML is not enabled. true false false + 740.0e3 ${DIN_LOC_ROOT}/atm/scream/tables/p3_lookup_table_1.dat-v4.1.1, ${DIN_LOC_ROOT}/atm/scream/tables/mu_r_table_vals.dat8, @@ -197,11 +208,70 @@ be lost if SCREAM_HACK_XML is not enabled. false + false + 0.001 + 0.04 + 2.65 + 0.02 + 1.0 + 1.0 + 1.0 + 1.0 + 0.5 + 7.0 + 0.1 + 0.1 + + + + + 0 + false + + TIME_DEPENDENT_3D_PROFILE + + + "no-file-given" + + + 0.0 + + + + + + + + + + false + + + + + 1,2 + 3,4 + 5,6 + 3,4 + 5,6 + + UNSET @@ -229,7 +299,7 @@ be lost if SCREAM_HACK_XML is not enabled. - h2o, co2, o3, n2o, co, ch4, o2, n2 + h2o, co2, o3, n2o, co, ch4, o2, n2 1807.851e-9 388.717e-6 323.141e-9 @@ -258,25 +328,49 @@ be lost if SCREAM_HACK_XML is not enabled. 3 3 4 - true + true false - false + false + + false + + + false + - (shoc,cldFraction,spa,p3) - (shoc,cldFraction,p3) + shoc,cldFraction,spa,p3 + tms,shoc,cldFraction,spa,p3 + shoc,cldFraction,p3 + tms,shoc,cldFraction,p3 24 - 6 - 3 - 3 - 1 + 12 + 6 + 6 + 2 1 5 + + 10 + + 1 + hours + + + + + - (mac_aero_mic,rrtmgp) + mac_aero_mic,rrtmgp @@ -311,14 +405,14 @@ be lost if SCREAM_HACK_XML is not enabled. UNSET - + ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_ne30np4pg2_x6t-SGH.c20210614.nc + ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_ne120np4pg2_x6t_20230404.nc ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_ne256np4pg2_x6t-SGH.c20210614.nc + ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_ne512np4pg2_x6t_20230404.nc ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_ne1024np4pg2_x6t-SGH.c20210614.nc - + ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc - ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_ne120np4pg2_16xdel2.nc - ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_ne512np4pg2_16xconsistentSGH_20190212_converted.nc ${DIN_LOC_ROOT}/atm/cam/topo/USGS_conusx4v1pg2_12x_consistentSGH_20200609.nc @@ -351,7 +445,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0 - 0.0,0.0 + 0.0,0.0 0.0 @@ -366,7 +460,7 @@ be lost if SCREAM_HACK_XML is not enabled. ${SRCROOT}/components/eamxx/data/scream_default_output.yaml ./${CASE}.scream - + ${REST_N} ${REST_OPTION} @@ -376,12 +470,17 @@ be lost if SCREAM_HACK_XML is not enabled. 0 - info + + info + false 1e-10 1e-14 Warning true + phis,landfrac @@ -452,6 +551,9 @@ be lost if SCREAM_HACK_XML is not enabled. 10 0 100.0 + + 0 2 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/bfbhash/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/bfbhash/shell_commands index c0b42727ba49..b962f5f962b9 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/bfbhash/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/bfbhash/shell_commands @@ -1 +1,2 @@ -./xmlchange --append SCREAM_ATMCHANGE_BUFFER='BfbHash=6' + +$CIMEROOT/../components/eamxx/scripts/atmchange BfbHash=1 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands new file mode 100644 index 000000000000..d3a4a39b668a --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands @@ -0,0 +1,2 @@ +$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +./xmlchange POSTRUN_SCRIPT="$CIMEROOT/../components/eamxx/tests/postrun/check_hashes_ers.py" diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/rad_frequency_2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/rad_frequency_2/shell_commands index 5ccb459798ed..d0abbbeb0c7f 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/rad_frequency_2/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/rad_frequency_2/shell_commands @@ -1 +1 @@ -./xmlchange SCREAM_ATMCHANGE_BUFFER='rad_frequency=2' +$CIMEROOT/../components/eamxx/scripts/atmchange rad_frequency=2 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_atmchange/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_atmchange/shell_commands index 0ebd594935b7..b7cd82b0c548 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_atmchange/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_atmchange/shell_commands @@ -1 +1,2 @@ -./xmlchange --append SCREAM_ATMCHANGE_BUFFER='cubed_sphere_map=42' + +$CIMEROOT/../components/eamxx/scripts/atmchange cubed_sphere_map=42 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands index 0496585c7d0d..04989a22796a 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands @@ -1 +1,7 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SMALL_KERNELS On' +$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b + +f=$(./xmlquery --value MACH) +if [ $f == chrysalis ]; then + ./xmlchange BATCH_COMMAND_FLAGS="--time 00:30:00 -p debug --account e3sm --exclude=chr-0512" +fi diff --git a/components/eamxx/cime_config/yaml_utils.py b/components/eamxx/cime_config/yaml_utils.py new file mode 100644 index 000000000000..73a5b13d09bb --- /dev/null +++ b/components/eamxx/cime_config/yaml_utils.py @@ -0,0 +1,72 @@ +# Add path to scream libs +import sys, os +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts")) + +from utils import ensure_yaml # pylint: disable=no-name-in-module +ensure_yaml() +import yaml + +############################################################################### +# These are types that we use to differentiate lists of ints,bools,floats,strings +# We can use these types to tell YAML how to write them to file, which ultimately +# means to simply add the proper tag to the yaml file +############################################################################### +class Array(list): + def __init__ (self, vals, t): + super().__init__(t(v) for v in vals) +class Bools(Array): + def __init__ (self,vals): + Array.__init__(self,vals,bool) +class Ints(Array): + def __init__ (self,vals): + Array.__init__(self,vals,int) +class Floats(Array): + def __init__ (self,vals): + Array.__init__(self,vals,float) +class Strings(Array): + def __init__ (self,vals): + Array.__init__(self,vals,str) + +############################################################################### +def make_array (vals,etype): +############################################################################### + if etype=="bool" or etype=="logical": + return Bools(vals) + elif etype=="int" or etype=="integer": + return Ints(vals) + elif etype=="float" or etype=="real": + return Floats(vals) + elif etype=="string" or etype=="file": + return Strings(vals) + else: + raise ValueError (f"Unsupported element type '{etype}' for arrays.") + +############################################################################### +def array_constructor(loader: yaml.SafeLoader, node: yaml.nodes.SequenceNode) -> list: +############################################################################### + entries = loader.construct_sequence(node) + if node.tag=="!bools": + return Bools(entries) + elif node.tag=="!ints": + return Ints(entries) + elif node.tag=="!floats": + return Floats(entries) + elif node.tag=="!strings": + return Strings(entries) + else: + raise ValueError(f"Invalid node tag={node.tag} for array constructor.") + +############################################################################### +def array_representer(dumper,array) -> yaml.nodes.SequenceNode: +############################################################################### + if isinstance(array,Bools): + return dumper.represent_sequence('!bools',array) + elif isinstance(array,Ints): + return dumper.represent_sequence('!ints',array) + elif isinstance(array,Floats): + return dumper.represent_sequence('!floats',array) + elif isinstance(array,Strings): + return dumper.represent_sequence('!strings',array) + else: + raise ValueError (f"Unsupported array type: {type(array)}") + diff --git a/components/eamxx/cmake/CompareNCFiles.cmake b/components/eamxx/cmake/CompareNCFiles.cmake new file mode 100644 index 000000000000..e17bc76b3a96 --- /dev/null +++ b/components/eamxx/cmake/CompareNCFiles.cmake @@ -0,0 +1,203 @@ +# Utility to create a test that compares two nc files +# Mandatory keyword arguments +# - TEST_NAME: the name to be given to the test +# - SRC_FILE: the name of the first nc file +# - TGT_FILE: the name of the second nc file +# Optional keyword arguments +# - LABELS: labels to attach to the created tests +# - FIXTURES_REQUIRED: list of fixtures required +function(CompareNCFiles) + # Parse keyword arguments + set (options) + set (args1v TEST_NAME SRC_FILE TGT_FILE) + set (argsMv LABELS FIXTURES_REQUIRED) + + cmake_parse_arguments(PARSE "${options}" "${args1v}" "${argsMv}" ${ARGN}) + CheckMacroArgs(CompareNCFilesFamily PARSE "${options}" "${args1v}" "${argsMv}") + + # Sanity checks + if (NOT PARSE_TEST_NAME) + message ("Error! CompareNCFilesPair requires the keyword argument TEST_NAME") + message (FATAL_ERROR "Aborting...") + endif() + if (NOT PARSE_SRC_FILE) + message ("Error! CompareNCFilesPair requires the keyword argument SRC_FILE") + message (FATAL_ERROR "Aborting...") + endif() + if (NOT PARSE_TGT_FILE) + message ("Error! CompareNCFilesPair requires the keyword argument TGT_FILE") + message (FATAL_ERROR "Aborting...") + endif() + + add_test ( + NAME ${PARSE_TEST_NAME} + COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${PARSE_SRC_FILE} ${PARSE_TGT_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + + # Set test properties, if needed + if (PARSE_LABELS) + set_tests_properties(${PARSE_TEST_NAME} PROPERTIES LABELS "${PARSE_LABELS}") + endif() + + # Set test fixtures, if needed + if (PARSE_FIXTURES_REQUIRED) + set_tests_properties(${PARSE_TEST_NAME} PROPERTIES FIXTURES_REQUIRED "${PARSE_FIXTURES_REQUIRED}") + endif() +endfunction() + +# This function is a more complex version of the one above: it creates tests +# to compare a set of files, which differ in their name by a simple substring. +# For instance, files generated with a different choice of a parameter + +# Mandatory keyword arguments +# - TEST_META_NAME: the base name to be given to the tests generated by this macro +# - FILE_META_NAME: the name of the files +# - MAGIC_STRING : the string that will be replaced with MAGIC_VALUES entries +# - MAGIC_VALUES : the values to be used to replace ${MAGIC_STRING} +# Optional keyword arguments +# - LABELS: labels to attach to the created tests +# - FIXTURES_REQUIRED: list of fixtures required +# Note: +# - TEST_META_NAME and FILE_META_NAME *MUST* contain the MAGIC_STRING +# - FIXTURES_REQUIRED *can* contain the MAGIC_STRING (but doesn't have to) +function (CompareNCFilesFamily) + # Parse keyword arguments + set (options) + set (args1v TEST_META_NAME FILE_META_NAME MAGIC_STRING) + set (argsMv MAGIC_VALUES LABELS FIXTURES_REQUIRED) + + cmake_parse_arguments(PARSE "${options}" "${args1v}" "${argsMv}" ${ARGN}) + CheckMacroArgs(CompareNCFilesFamily PARSE "${options}" "${args1v}" "${argsMv}") + + # Sanity checks + if (NOT PARSE_TEST_META_NAME) + message ("Error! CompareNCFilesFamily requires the keyword argument TEST_META_NAME") + message (FATAL_ERROR "Aborting...") + endif() + if (NOT PARSE_FILE_META_NAME) + message ("Error! CompareNCFilesFamily requires the keyword argument FILE_META_NAME") + message (FATAL_ERROR "Aborting...") + endif() + if (NOT PARSE_MAGIC_STRING) + message ("Error! CompareNCFilesFamily requires the keyword argument MAGIC_STRING") + message (FATAL_ERROR "Aborting...") + endif() + if (NOT PARSE_MAGIC_VALUES) + message ("Error! CompareNCFilesFamily requires the keyword argument MAGIC_VALUES") + message (FATAL_ERROR "Aborting...") + endif() + if (NOT PARSE_TEST_META_NAME MATCHES ${PARSE_MAGIC_STRING}) + message ("Error! MAGIC_STRING not contained in TEST_META_NAME.") + message (" MAGIC_STRING: ${PARSE_MAGIC_STRING}") + message (" TEST_META_NAME: ${PARSE_TEST_META_NAME}") + message (FATAL_ERROR "Aborting...") + endif() + if (NOT PARSE_FILE_META_NAME MATCHES ${PARSE_MAGIC_STRING}) + message ("Error! MAGIC_STRING not contained in FILE_META_NAME.") + message (" MAGIC_STRING: ${PARSE_MAGIC_STRING}") + message (" FILE_META_NAME: ${PARSE_FILE_META_NAME}") + message (FATAL_ERROR "Aborting...") + endif() + + # Ensure cprnc is built + include (BuildCprnc) + BuildCprnc() + + # Remove first entry of magic values. Compare all other entries against this + list (POP_FRONT PARSE_MAGIC_VALUES first) + string (REPLACE "${PARSE_MAGIC_STRING}" "${first}" TGT_FILE ${PARSE_FILE_META_NAME}) + + # FIXTURES_REQUIRED *can* also contain the magic string + foreach (item IN LISTS PARSE_MAGIC_VALUES) + # Expand the magic string in src file + string (REPLACE ${PARSE_MAGIC_STRING} ${item} SRC_FILE ${PARSE_FILE_META_NAME}) + + # Create the test. Also the test base name may contain the magic string + string (REPLACE ${PARSE_MAGIC_STRING} ${item} TEST_NAME ${PARSE_TEST_META_NAME}) + + add_test ( + NAME ${TEST_NAME} + COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + + # Set test properties, if needed + if (PARSE_LABELS) + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "${PARSE_LABELS}") + endif() + + # Set test fixtures, if needed + if (PARSE_FIXTURES_REQUIRED) + set (TMP_LIST ${PARSE_FIXTURES_REQUIRED}) + list (TRANSFORM TMP_LIST REPLACE "${PARSE_MAGIC_STRING}" ${item}) + + set_tests_properties(${TEST_NAME} PROPERTIES + FIXTURES_REQUIRED "${TMP_LIST}") + endif() + endforeach() +endfunction (CompareNCFilesFamily) + +# A version of the above tailored for PEM-like comparisons, where the family of NC files +# corresponds to runs using different number of MPI ranks +function (CompareNCFilesFamilyMpi) + # Parse keyword arguments + set (options) + set (args1v TEST_BASE_NAME FILE_META_NAME) + set (argsMv MPI_RANKS LABELS META_FIXTURES_REQUIRED) + cmake_parse_arguments(PARSE "${options}" "${args1v}" "${argsMv}" ${ARGN}) + CheckMacroArgs(CompareNCFilesFamily PARSE "${options}" "${args1v}" "${argsMv}") + + if (NOT PARSE_TEST_BASE_NAME) + message ("Error! CompareNCFilesFamilyMpi requires the keyword argument TEST_BASE_NAME") + message (FATAL_ERROR "Aborting...") + endif() + + # Grab the args for the MPI_RANKS range specs. This follows the same convention of CreateUnitTest: + # - 1 value: start==end + # - 2 values: + # - 3 values: + list (LENGTH PARSE_MPI_RANKS NUM_MPI_RANK_ARGS) + + if (NUM_MPI_RANK_ARGS EQUAL 2) + list (GET PARSE_MPI_RANKS 0 BEG) + list (GET PARSE_MPI_RANKS 1 END) + set (INC 1) + elseif(NUM_MPI_RANK_ARGS EQUAL 3) + list (GET PARSE_MPI_RANKS 0 BEG) + list (GET PARSE_MPI_RANKS 1 END) + list (GET PARSE_MPI_RANKS 2 INC) + else() + message ("CompareNCFilesFamilyMpi requires 2 or 3 values for the keyword argument MPI_RANKS") + message (" Input values: ${PARSE_MPI_RANKS}") + message (FATAL_ERROR "Aborting...") + endif() + + # Create the range + CreateRange(MpiRanks ${BEG} ${END} ${INC}) + + # The input META_FIXTURES_REQUIRED is a required argument, which *MUST* contain the "MPIRANKS" string. + # We assume for each rank N, there is a test with FIXTURE_SETUP set to that string (with MPIRANKS=N). + if (NOT PARSE_META_FIXTURES_REQUIRED) + message ("Missing value for the mandatory META_FIXTURES_REQUIRED keyword argument.") + message (FATAL_ERROR "Aborting...") + endif() + if (NOT PARSE_META_FIXTURES_REQUIRED MATCHES "MPIRANKS") + message ("Error! MPIRANKS string not contained in META_FIXTURES_REQUIRED.") + message (" META_FIXTURES_REQUIRED: ${META_FIXTURES_REQUIRED}") + message (FATAL_ERROR "Aborting...") + endif() + + # Each comparison is between rank=BEG and rank=N, so we need two fixtures, one of which + # has a predefined value for MPIRANKS. + string (REPLACE "MPIRANKS" ${BEG} REQUIRED_FIXTURES "${PARSE_META_FIXTURES_REQUIRED}") + list (APPEND REQUIRED_FIXTURES "${PARSE_META_FIXTURES_REQUIRED}") + + # Call the function above + CompareNCFilesFamily( + TEST_META_NAME ${PARSE_TEST_BASE_NAME}_npMPIRANKS_vs_np${BEG} + FILE_META_NAME ${PARSE_FILE_META_NAME} + MAGIC_STRING "MPIRANKS" + MAGIC_VALUES ${MpiRanks} + LABELS ${PARSE_LABELS} PEM + FIXTURES_REQUIRED ${REQUIRED_FIXTURES} + ) +endfunction() diff --git a/components/eamxx/cmake/ScreamUtils.cmake b/components/eamxx/cmake/ScreamUtils.cmake index ae21931f5bce..44e3e79ce0d5 100644 --- a/components/eamxx/cmake/ScreamUtils.cmake +++ b/components/eamxx/cmake/ScreamUtils.cmake @@ -2,6 +2,44 @@ include(CMakeParseArguments) # Needed for backwards compatibility include(EkatCreateUnitTest) include(EkatUtils) +# Create a list containing a range of integers +function (CreateRange resultVar BEG END) + set(options SKIP_FIRST SKIP_LAST) + set(arg1v INC) + set(argMv) + cmake_parse_arguments(CR "${options}" "${arg1v}" "${argMv}" ${ARGN}) + + # Compute beg/end/inc based on input args + if (CR_SKIP_FIRST) + math(EXPR BEG "${BEG}+1") + endif() + if (CR_SKIP_LAST) + math(EXPR END "${END}-1") + endif() + if (NOT CR_INC) + set (CR_INC 1) + endif() + + # Sanity check + if (NOT CR_INC GREATER 0) + message (FATAL_ERROR "INC must be a positive integer") + endif() + if (BEG GREATER END) + message (FATAL_ERROR "BEG is larger than END") + endif() + + # Create range list + set (res_list) + set (N ${BEG}) + while (NOT N GREATER END) + list (APPEND res_list ${N}) + math (EXPR N "${N}+${CR_INC}") + endwhile() + + # Set in parent scope + set (${resultVar} ${res_list} PARENT_SCOPE) +endfunction() + # This function takes the following arguments: # - test_name: the base name of the test. We create an executable with this name # - test_srcs: a list of src files for the executable. @@ -45,40 +83,27 @@ set(SCREAM_CUT_TEST_MV_ARGS ${CUT_TEST_MV_ARGS}) # Scream always excludes the ekat test session since it has its own list(REMOVE_ITEM SCREAM_CUT_EXEC_OPTIONS EXCLUDE_TEST_SESSION) -# Libs are a position arg for SCREAM, not an optional arg like in EKAT -list(REMOVE_ITEM SCREAM_CUT_EXEC_MV_ARGS LIBS) - ############################################################################### -function(CreateUnitTestExec exec_name test_srcs scream_libs) +function(CreateUnitTestExec exec_name test_srcs) ############################################################################### - cmake_parse_arguments(cute "${SCREAM_CUT_EXEC_OPTIONS}" "${SCREAM_CUT_EXEC_1V_ARGS}" "${SCREAM_CUT_EXEC_MV_ARGS}" ${ARGN}) - CheckMacroArgs(CreateUnitTestExec cute "${SCREAM_CUT_EXEC_OPTIONS}" "${SCREAM_CUT_EXEC_1V_ARGS}" "${SCREAM_CUT_EXEC_MV_ARGS}") - - separate_cut_arguments(cute "${SCREAM_CUT_EXEC_OPTIONS}" "${SCREAM_CUT_EXEC_1V_ARGS}" "${SCREAM_CUT_EXEC_MV_ARGS}" options) - - set(TEST_INCLUDE_DIRS - ${SCREAM_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ) - - set(test_libs "${scream_libs};scream_test_support") - list(APPEND test_libs "${SCREAM_TPL_LIBRARIES}") - - if (SCREAM_Fortran_FLAGS) - list(APPEND options COMPILER_F_FLAGS ${SCREAM_Fortran_FLAGS}) - endif () - - EkatCreateUnitTestExec("${exec_name}" "${test_srcs}" ${options} - EXCLUDE_TEST_SESSION LIBS ${test_libs} INCLUDE_DIRS ${TEST_INCLUDE_DIRS}) - + # Call Ekat function, with a couple of extra params + EkatCreateUnitTestExec("${exec_name}" "${test_srcs}" ${ARGN} + EXCLUDE_TEST_SESSION LIBS scream_share scream_test_support) endfunction(CreateUnitTestExec) +############################################################################### +function(CreateADUnitTestExec exec_name) +############################################################################### + # Call the function above specifying some params + CreateUnitTestExec("${exec_name}" "${SCREAM_SRC_DIR}/share/util/eamxx_ad_test.cpp" + LIBS scream_control scream_io diagnostics ${ARGN}) +endfunction(CreateADUnitTestExec) + ############################################################################### function(CreateUnitTestFromExec test_name test_exec) ############################################################################### cmake_parse_arguments(cutfe "${SCREAM_CUT_TEST_OPTIONS}" "${SCREAM_CUT_TEST_1V_ARGS}" "${SCREAM_CUT_TEST_MV_ARGS}" ${ARGN}) - CheckMacroArgs(CreateUnitTestExec cutfe "${SCREAM_CUT_TEST_OPTIONS}" "${SCREAM_CUT_TEST_1V_ARGS}" "${SCREAM_CUT_TEST_MV_ARGS}") + CheckMacroArgs(CreateUnitTestFromExec cutfe "${SCREAM_CUT_TEST_OPTIONS}" "${SCREAM_CUT_TEST_1V_ARGS}" "${SCREAM_CUT_TEST_MV_ARGS}") # # If asking for mpi/omp ranks/threads, verify we stay below the max number of threads @@ -132,7 +157,7 @@ function(CreateUnitTestFromExec test_name test_exec) endfunction(CreateUnitTestFromExec) ############################################################################### -function(CreateUnitTest test_name test_srcs scream_libs) +function(CreateUnitTest test_name test_srcs) ############################################################################### set(options ${SCREAM_CUT_EXEC_OPTIONS} ${SCREAM_CUT_TEST_OPTIONS}) set(oneValueArgs ${SCREAM_CUT_EXEC_1V_ARGS} ${SCREAM_CUT_TEST_1V_ARGS}) @@ -147,7 +172,7 @@ function(CreateUnitTest test_name test_srcs scream_libs) #------------------------------# separate_cut_arguments(cut "${SCREAM_CUT_EXEC_OPTIONS}" "${SCREAM_CUT_EXEC_1V_ARGS}" "${SCREAM_CUT_EXEC_MV_ARGS}" options_ExecPhase) - CreateUnitTestExec("${test_name}" "${test_srcs}" "${scream_libs}" ${options_ExecPhase}) + CreateUnitTestExec("${test_name}" "${test_srcs}" ${options_ExecPhase}) #------------------------------# # Create Tests Phase # @@ -158,6 +183,15 @@ function(CreateUnitTest test_name test_srcs scream_libs) endfunction(CreateUnitTest) +############################################################################### +function(CreateADUnitTest test_name) +############################################################################### + + # Call the function above specifying some params + CreateUnitTest("${test_name}" "${SCREAM_SRC_DIR}/share/util/eamxx_ad_test.cpp" + LABELS driver LIBS scream_control scream_io diagnostics ${ARGN}) +endfunction(CreateADUnitTest) + ############################################################################### function(GetInputFile src_path) ############################################################################### diff --git a/components/eamxx/cmake/machine-files/alvarez.cmake b/components/eamxx/cmake/machine-files/alvarez.cmake index 6df22e6fe08c..037da48eaf03 100644 --- a/components/eamxx/cmake/machine-files/alvarez.cmake +++ b/components/eamxx/cmake/machine-files/alvarez.cmake @@ -1,18 +1,8 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -#message(STATUS "alvarez PROJECT_NAME=${PROJECT_NAME} USE_CUDA=${USE_CUDA} KOKKOS_ENABLE_CUDA=${KOKKOS_ENABLE_CUDA}") - include (${EKAT_MACH_FILES_PATH}/kokkos/amd-zen3.cmake) -if ("${PROJECT_NAME}" STREQUAL "E3SM") - if (BUILD_THREADED) - include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) - else() - include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) - endif() -else() - include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -endif() +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) diff --git a/components/eamxx/cmake/machine-files/compy.cmake b/components/eamxx/cmake/machine-files/compy.cmake index ebd53132699b..1f156b0546c0 100644 --- a/components/eamxx/cmake/machine-files/compy.cmake +++ b/components/eamxx/cmake/machine-files/compy.cmake @@ -6,7 +6,5 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/intel-skx.cmake) include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) -set (NetCDF_PATH /share/apps/netcdf/4.6.3/gcc/8.1.0 CACHE STRING "") - #Compy SLURM specific settings set(EKAT_MPI_NP_FLAG "-p short -n" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/cori-knl.cmake b/components/eamxx/cmake/machine-files/cori-knl.cmake index f1a2bca491e2..11296f343748 100644 --- a/components/eamxx/cmake/machine-files/cori-knl.cmake +++ b/components/eamxx/cmake/machine-files/cori-knl.cmake @@ -3,23 +3,14 @@ common_setup() # Load knl arch and openmp backend for kokkos include (${EKAT_MACH_FILES_PATH}/kokkos/intel-knl.cmake) - -if ("${PROJECT_NAME}" STREQUAL "E3SM") - if (BUILD_THREADED) - include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) - else() - include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) - endif() -else() - include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -endif() +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) if ("${PROJECT_NAME}" STREQUAL "E3SM") if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10) - set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) # only works with gnu v10 and above + set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) # only works with gnu v10 and above endif() endif() else() diff --git a/components/eamxx/cmake/machine-files/crusher-scream-gpu.cmake b/components/eamxx/cmake/machine-files/crusher-scream-gpu.cmake index c35c9fd66bfe..391a9b8d6880 100644 --- a/components/eamxx/cmake/machine-files/crusher-scream-gpu.cmake +++ b/components/eamxx/cmake/machine-files/crusher-scream-gpu.cmake @@ -2,7 +2,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() #serial is needed, but maybe it is always on? -#include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) include (${EKAT_MACH_FILES_PATH}/kokkos/mi250.cmake) include (${EKAT_MACH_FILES_PATH}/kokkos/hip.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) diff --git a/components/eamxx/cmake/machine-files/docker-scream.cmake b/components/eamxx/cmake/machine-files/docker-scream.cmake index 0287c5d399f7..87d507fef20f 100644 --- a/components/eamxx/cmake/machine-files/docker-scream.cmake +++ b/components/eamxx/cmake/machine-files/docker-scream.cmake @@ -8,7 +8,7 @@ set(BLAS_LIBRARIES /opt/conda/lib/libblas.so CACHE STRING "") set(LAPACK_LIBRARIES /opt/conda/lib/liblapack.so CACHE STRING "") set(SCREAM_INPUT_ROOT "/storage/inputdata/" CACHE STRING "") set(PYBIND11_PYTHON_VERSION 3.9 CACHE STRING "") -set(RUN_ML_CORRECTION_TEST TRUE CACHE BOOL "") +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) if ("${PROJECT_NAME}" STREQUAL "E3SM") if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") @@ -18,4 +18,4 @@ if ("${PROJECT_NAME}" STREQUAL "E3SM") endif() else() set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) # only works with gnu v10 and above -endif() \ No newline at end of file +endif() diff --git a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake new file mode 100644 index 000000000000..14d1d501160d --- /dev/null +++ b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake @@ -0,0 +1,9 @@ +set (EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) + +include (${EKAT_MACH_FILES_PATH}/kokkos/mi250.cmake) +include (${EKAT_MACH_FILES_PATH}/kokkos/hip.cmake) + +set(SCREAM_MPIRUN_EXE "srun" CACHE STRING "") +set(SCREAM_MACHINE "frontier-scream-gpu" CACHE STRING "") + +set(CMAKE_CXX_FLAGS "--amdgpu-target=gfx90a -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/gcp.cmake b/components/eamxx/cmake/machine-files/gcp.cmake index fe105682b1d9..ecfa8d7e9604 100644 --- a/components/eamxx/cmake/machine-files/gcp.cmake +++ b/components/eamxx/cmake/machine-files/gcp.cmake @@ -1,19 +1,8 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -#message(STATUS "gcp PROJECT_NAME=${PROJECT_NAME} USE_CUDA=${USE_CUDA} KOKKOS_ENABLE_CUDA=${KOKKOS_ENABLE_CUDA}") # use default backend? - -if ("${PROJECT_NAME}" STREQUAL "E3SM") - if (BUILD_THREADED) - include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) - else() - include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) - endif() -else() - include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -endif() - +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/lassen.cmake b/components/eamxx/cmake/machine-files/lassen.cmake index a709bba15689..36b69c7f0253 100644 --- a/components/eamxx/cmake/machine-files/lassen.cmake +++ b/components/eamxx/cmake/machine-files/lassen.cmake @@ -1,8 +1,9 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -set(NetCDF_Fortran_PATH /usr/gdata/climdat/libs/netcdf-fortran/install/lassen/fortran CACHE STRING "") -set(BLAS_LIBRARIES /usr/gdata/climdat/libs/blas/libblas.a CACHE STRING "") -set(LAPACK_LIBRARIES /usr/gdata/climdat/libs/lapack/liblapack.a CACHE STRING "") +set(NetCDF_PATH /usr/gdata/climdat/netcdf CACHE STRING "") +set(NetCDF_Fortran_PATH /usr/gdata/climdat/netcdf CACHE STRING "") +set(LAPACK_LIBRARIES /usr/lib64/liblapack.so CACHE STRING "") +set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata/" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/mappy.cmake b/components/eamxx/cmake/machine-files/mappy.cmake index 86a2fb1d5302..7c1fc8cf25ea 100644 --- a/components/eamxx/cmake/machine-files/mappy.cmake +++ b/components/eamxx/cmake/machine-files/mappy.cmake @@ -1,2 +1,3 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() +set(PYTHON_EXECUTABLE "/ascldap/users/jgfouca/packages/Python-3.8.5/bin/python3.8" CACHE STRING "" FORCE) \ No newline at end of file diff --git a/components/eamxx/cmake/machine-files/pm-cpu.cmake b/components/eamxx/cmake/machine-files/pm-cpu.cmake index 3c32d23dfbff..ef66da562f02 100644 --- a/components/eamxx/cmake/machine-files/pm-cpu.cmake +++ b/components/eamxx/cmake/machine-files/pm-cpu.cmake @@ -1,17 +1,8 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -if ("${PROJECT_NAME}" STREQUAL "E3SM") - if (BUILD_THREADED) - include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) - #message(STATUS, "pm-cpu openmp BUILD_THREADED=${BUILD_THREADED}") - else() - include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) - #message(STATUS, "pm-cpu serial BUILD_THREADED=${BUILD_THREADED}") - endif() -else() - include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -endif() +include (${EKAT_MACH_FILES_PATH}/kokkos/amd-zen3.cmake) +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/quartz-intel.cmake b/components/eamxx/cmake/machine-files/quartz-intel.cmake index cefda8437da4..753c782702db 100644 --- a/components/eamxx/cmake/machine-files/quartz-intel.cmake +++ b/components/eamxx/cmake/machine-files/quartz-intel.cmake @@ -1,3 +1,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/quartz.cmake) -set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/ -qmkl" CACHE STRING "" FORCE) -set(RUN_ML_CORRECTION_TEST TRUE CACHE BOOL "") +set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64 -qmkl" CACHE STRING "" FORCE) +set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE STRING "" FORCE) +set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) +set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) +execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/quartz.cmake b/components/eamxx/cmake/machine-files/quartz.cmake index 24c97078475c..ee9a3dcbffd3 100644 --- a/components/eamxx/cmake/machine-files/quartz.cmake +++ b/components/eamxx/cmake/machine-files/quartz.cmake @@ -9,7 +9,7 @@ option(Kokkos_ARCH_BDW "" ON) #if COMPILER is not defined, should be running standalone with quartz-intel or quartz-gcc if ("${COMPILER}" STREQUAL "intel") - set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/ -qmkl" CACHE STRING "" FORCE) + set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64/ -qmkl" CACHE STRING "" FORCE) elseif ("${COMPILER}" STREQUAL "gnu") message(WARNING "You are using an unsupported e3sm compiler. For supported quartz compilers run ./${E3SM_ROOT}/cime/scripts/query_config --machines quartz") set(CMAKE_CXX_FLAGS "-w" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/ruby-intel.cmake b/components/eamxx/cmake/machine-files/ruby-intel.cmake index 5ebae48a9285..63fff478fdaf 100644 --- a/components/eamxx/cmake/machine-files/ruby-intel.cmake +++ b/components/eamxx/cmake/machine-files/ruby-intel.cmake @@ -1,5 +1,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/ruby.cmake) -set(CMAKE_CXX_FLAGS "-w -cxxlib=/usr/tce/packages/gcc/gcc-8.3.1/rh" CACHE STRING "" FORCE) -set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/ -mkl" CACHE STRING "" FORCE) -set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.8.2/bin/python3" CACHE STRING "" FORCE) -set(RUN_ML_CORRECTION_TEST TRUE CACHE BOOL "") +set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64/ -qmkl" CACHE STRING "" FORCE) +set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE STRING "" FORCE) +set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) +set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) +execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/ruby.cmake b/components/eamxx/cmake/machine-files/ruby.cmake index 20a7eb008444..d0a9de4baf4b 100644 --- a/components/eamxx/cmake/machine-files/ruby.cmake +++ b/components/eamxx/cmake/machine-files/ruby.cmake @@ -12,6 +12,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) -set(CMAKE_CXX_FLAGS "-w -cxxlib=/usr/tce/packages/gcc/gcc-8.3.1/rh" CACHE STRING "" FORCE) -set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/ -mkl" CACHE STRING "" FORCE) - +set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/weaver.cmake b/components/eamxx/cmake/machine-files/weaver.cmake index fa948b80003c..cf8251b44877 100644 --- a/components/eamxx/cmake/machine-files/weaver.cmake +++ b/components/eamxx/cmake/machine-files/weaver.cmake @@ -1,8 +1,9 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -set (BLAS_LIBRARIES /ascldap/users/projects/e3sm/scream/libs/openblas/install/weaver/gcc/8.5.0/lib/libopenblas.so CACHE STRING "") -set (LAPACK_LIBRARIES /ascldap/users/projects/e3sm/scream/libs/openblas/install/weaver/gcc/8.5.0/lib/libopenblas.so CACHE STRING "") +set (BLAS_LIBRARIES $ENV{NETLIB_LAPACK_ROOT}/lib64/libblas.so CACHE STRING "") +set (LAPACK_LIBRARIES $ENV{NETLIB_LAPACK_ROOT}/lib64/liblapack.so CACHE STRING "") set(SCREAM_INPUT_ROOT "/home/projects/e3sm/scream/data" CACHE STRING "") -set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) +#set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) set(HOMMEXX_CUDA_MAX_WARP_PER_TEAM 8 CACHE STRING "") diff --git a/components/eamxx/cmake/tpls/CsmShare.cmake b/components/eamxx/cmake/tpls/CsmShare.cmake index 3faa98d3ec9c..fd682d0ca4e9 100644 --- a/components/eamxx/cmake/tpls/CsmShare.cmake +++ b/components/eamxx/cmake/tpls/CsmShare.cmake @@ -1,55 +1,25 @@ macro (CreateCsmShareTarget) - if (TARGET csm_share) - message (FATAL_ERROR "Error! The target csm_share already exists!") - endif() - if (SCREAM_CIME_BUILD) - # Some sanity checks - if (NOT DEFINED INSTALL_SHAREDPATH) - message (FATAL_ERROR "Error! The cmake variable 'INSTALL_SHAREDPATH' is not defined.") - endif () - if (NOT DEFINED COMP_INTERFACE) - message (FATAL_ERROR "Error! The cmake variable 'COMP_INTERFACE' is not defined.") - endif () - if (NOT DEFINED NINST_VALUE) - message (FATAL_ERROR "Error! The cmake variable 'NINST_VALUE' is not defined.") - endif () - - # If we didn't already parse this script, create imported target - if (NOT TARGET csm_share) - - # Build the name of the path where libcsm_share should be located - if (USE_ESMF_LIB) - set(ESMFDIR "esmf") - else() - set(ESMFDIR "noesmf") - endif() - set(CSM_SHARE "${INSTALL_SHAREDPATH}/${COMP_INTERFACE}/${ESMFDIR}/${NINST_VALUE}/csm_share") + find_package(CsmShare REQUIRED) - # Look for libcsm_share in the complex path we built above - find_library(CSM_SHARE_LIB csm_share REQUIRED PATHS ${CSM_SHARE}) - - # Create the interface library, and set target properties - add_library (csm_share INTERFACE) - target_link_libraries (csm_share INTERFACE ${CSM_SHARE_LIB}) - target_include_directories(csm_share INTERFACE ${CSM_SHARE}) - - # Link against piof - target_link_libraries(csm_share INTERFACE piof) - endif () else() # Build csm_share library manually + if (TARGET csm_share) + message (FATAL_ERROR "Error! The target csm_share already exists!") + endif() # Set variables needed for processing genf90 templates set(CIMEROOT ${SCREAM_BASE_DIR}/../../cime) list(APPEND CMAKE_MODULE_PATH ${CIMEROOT}/CIME/non_py/src/CMake) - set(GENF90 ${CIMEROOT}/CIME/non_py/externals/genf90/genf90.pl) + # Setting GENF90_PATH here will prevent cprnc from trying to redefine the genf90 target + set(GENF90_PATH ${CIMEROOT}/CIME/non_py/externals/genf90) + set(GENF90 ${GENF90_PATH}/genf90.pl) set(ENABLE_GENF90 True) include(genf90_utils) include(Sourcelist_utils) # GENF90_SOURCE lists source files we will need to run through the genf90 perl script - set (GENF90_SOURCE + set (GENF90_SOURCE ${SCREAM_BASE_DIR}/../../share/util/shr_infnan_mod.F90.in ${SCREAM_BASE_DIR}/../../share/util/shr_assert_mod.F90.in ) diff --git a/components/eamxx/cmake/tpls/GPTL.cmake b/components/eamxx/cmake/tpls/GPTL.cmake deleted file mode 100644 index 0442a136eda6..000000000000 --- a/components/eamxx/cmake/tpls/GPTL.cmake +++ /dev/null @@ -1,25 +0,0 @@ -macro (CreateGPTLTarget) - # Sanity check - if (TARGET gptl) - # We should not call this macro twice - message (FATAL_ERROR "The GPTL target was already created!") - endif() - - if (SCREAM_CIME_BUILD) - # Some sanity checks - if (NOT DEFINED INSTALL_SHAREDPATH) - message (FATAL_ERROR "Error! The cmake variable 'INSTALL_SHAREDPATH' is not defined.") - endif () - - # Look for libgptl in INSTALL_SHAREDPATH/lib - find_library(GPTL_LIB gptl REQUIRED PATHS ${INSTALL_SHAREDPATH}/lib) - - # Create the imported target that scream targets can link to - add_library (gptl INTERFACE) - target_link_libraries (gptl INTERFACE ${GPTL_LIB}) - target_include_directories (gptl INTERFACE ${INSTALL_SHAREDPATH}/include) - if (NOT MPILIB STREQUAL "mpi-serial") - target_compile_definitions (gptl INTERFACE HAVE_MPI) - endif() - endif () -endmacro() diff --git a/components/eamxx/cmake/tpls/GetNetcdfLibs.cmake b/components/eamxx/cmake/tpls/GetNetcdfLibs.cmake deleted file mode 100644 index 6202577468d4..000000000000 --- a/components/eamxx/cmake/tpls/GetNetcdfLibs.cmake +++ /dev/null @@ -1,64 +0,0 @@ -# Use Macros.cmake to get info on netcdf paths. -# Note: the inputs are supposed to be *the name* of the variables storing the result -# Note: Keep this a FUNCTION, not a MACRO, to avoid polluting the calling scope -# with all the stuff from Macros.cmake -function (GetNetcdfLibs) - # Sanity check - if (NOT SCREAM_CIME_BUILD) - message (FATAL_ERROR "Error! Do not call 'GetNetcdfPaths' in a non-CIME build.\n") - endif () - - # Load variables set by CIME - include(${CASEROOT}/Macros.cmake) - - # Pnetcdf is optional, and only if not running serial - if (NOT MPILIB STREQUAL mpi-serial) - if (PNETCDF_PATH) - find_library(pnetcdf_lib pnetcdf REQUIRED PATHS ${PNETCDF_PATH}/lib) - set (pnetcdf_lib ${pnetcdf_lib} PARENT_SCOPE) - find_path (pnetcdf_incdir pnetcdf.h REQUIRED PATHS ${PNETCDF_PATH}/include) - endif() - endif() - - if (NETCDF_C_PATH) - # Sanity checks - if (NOT NETCDF_FORTRAN_PATH) - message(FATAL_ERROR "NETCDF_C_PATH specified without NETCDF_FORTRAN_PATH") - endif() - if (NOT EXISTS ${NETCDF_C_PATH}/lib AND NOT EXISTS ${NETCDF_C_PATH}/lib64) - message(FATAL_ERROR "NETCDF_C_PATH does not contain a lib or lib64 directory") - endif () - if (NOT EXISTS ${NETCDF_FORTRAN_PATH}/lib AND NOT EXISTS ${NETCDF_FORTRAN_PATH}/lib64) - message(FATAL_ERROR "NETCDF_FORTRAN_PATH does not contain a lib or lib64 directory") - endif () - - # Find the libraries - find_library(netcdf_c_lib netcdf REQUIRED PATHS ${NETCDF_C_PATH}/lib ${NETCDF_C_PATH}/lib64) - find_library(netcdf_f_lib netcdff REQUIRED PATHS ${NETCDF_FORTRAN_PATH}/lib ${NETCDF_FORTRAN_PATH}/lib64) - find_path (netcdf_c_incdir netcdf.h REQUIRED PATHS ${NETCDF_C_PATH}/include) - find_path (netcdf_f_incdir netcdf.inc REQUIRED PATHS ${NETCDF_FORTRAN_PATH}/include) - - elseif (NETCDF_FORTRAN_PATH) - message(FATAL_ERROR "NETCDF_FORTRAN_PATH specified without NETCDF_C_PATH") - elseif (NETCDF_PATH) - - # Sanity checks - if (NOT EXISTS ${NETCDF_PATH}/lib AND NOT EXISTS ${NETCDF_PATH}/lib64) - message(FATAL_ERROR "NETCDF_PATH does not contain a lib or lib64 directory") - endif () - - find_library(netcdf_c_lib netcdf REQUIRED PATHS ${NETCDF_PATH}/lib ${NETCDF_PATH}/lib64) - find_library(netcdf_f_lib netcdff REQUIRED PATHS ${NETCDF_PATH}/lib ${NETCDF_PATH}/lib64) - find_path (netcdf_c_incdir netcdf.h REQUIRED PATHS ${NETCDF_PATH}/include) - find_path (netcdf_f_incdir netcdf.inc REQUIRED PATHS ${NETCDF_PATH}/include) - else() - message(FATAL_ERROR "NETCDF not found: Define NETCDF_PATH or NETCDF_C_PATH and NETCDF_FORTRAN_PATH in config_machines.xml or config_compilers.xml") - endif() - set (pnetcdf_lib ${pnetcdf_lib} PARENT_SCOPE) - set (netcdf_c_lib ${netcdf_c_lib} PARENT_SCOPE) - set (netcdf_f_lib ${netcdf_f_lib} PARENT_SCOPE) - set (pnetcdf_incdir ${pnetcdf_incdir} PARENT_SCOPE) - set (netcdf_c_incdir ${netcdf_c_incdir} PARENT_SCOPE) - set (netcdf_f_incdir ${netcdf_f_incdir} PARENT_SCOPE) - -endfunction () diff --git a/components/eamxx/cmake/tpls/Mct.cmake b/components/eamxx/cmake/tpls/Mct.cmake deleted file mode 100644 index 1650bec4dab8..000000000000 --- a/components/eamxx/cmake/tpls/Mct.cmake +++ /dev/null @@ -1,26 +0,0 @@ -macro (CreateMctTarget) - - # Some sanity checks - if (NOT SCREAM_CIME_BUILD) - message (FATAL_ERROR "Error! You should need the mct target only in CIME builds") - endif () - if (NOT DEFINED INSTALL_SHAREDPATH) - message (FATAL_ERROR "Error! The cmake variable 'INSTALL_SHAREDPATH' is not defined.") - endif () - - if (TARGET mct) - # We should not call this macro twice - message (FATAL_ERROR "The mct target was already created!") - endif() - - # Look for libmct in INSTALL_SHAREDPATH/lib - find_library(MCT_LIB mct REQUIRED PATHS ${INSTALL_SHAREDPATH}/lib) - - # Create the interface library, and set target properties - add_library(mct INTERFACE) - target_link_libraries(mct INTERFACE ${MCT_LIB}) - target_include_directories(mct INTERFACE ${INSTALL_SHAREDPATH}/include) - - # Link against csm_share - target_link_libraries(mct INTERFACE csm_share) -endmacro() diff --git a/components/eamxx/cmake/tpls/Scorpio.cmake b/components/eamxx/cmake/tpls/Scorpio.cmake index b54aa2689a66..db1746298413 100644 --- a/components/eamxx/cmake/tpls/Scorpio.cmake +++ b/components/eamxx/cmake/tpls/Scorpio.cmake @@ -3,8 +3,6 @@ set (E3SM_EXTERNALS_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../../externals CACHE INTERNAL "") set (SCREAM_TPLS_MODULE_DIR ${CMAKE_CURRENT_LIST_DIR} CACHE INTERNAL "") -include (${SCREAM_TPLS_MODULE_DIR}/GPTL.cmake) -include (${SCREAM_TPLS_MODULE_DIR}/GetNetcdfLibs.cmake) macro (CreateScorpioTargets) @@ -15,55 +13,21 @@ macro (CreateScorpioTargets) endif() if (SCREAM_CIME_BUILD) - # For CIME builds, we simply wrap the already built pioc/piof libs into a cmake target - if (NOT DEFINED INSTALL_SHAREDPATH) - message (FATAL_ERROR "Error! The cmake variable 'INSTALL_SHAREDPATH' is not defined.") - endif () + find_package(PIO REQUIRED) - set(SCORPIO_LIB_DIR ${INSTALL_SHAREDPATH}/lib) - set(SCORPIO_INC_DIR ${INSTALL_SHAREDPATH}/include) - set(CSM_SHR_INCLUDE ${INSTALL_SHAREDPATH}/${COMP_INTERFACE}/noesmf/${NINST_VALUE}/include) - - # Look for pioc deps. We will have to link them to the pioc target, so that cmake will - # propagate them to any downstream target linking against pioc - CreateGPTLTarget() - GetNetcdfLibs() - - ###################### - # PIOc # - ###################### - - # Look for pioc in INSTALL_SHAREDPATH/lib - find_library(SCORPIO_C_LIB pioc REQUIRED PATHS ${SCORPIO_LIB_DIR}) - - # Create the interface library, and set target properties add_library (pioc INTERFACE) - target_link_libraries (pioc INTERFACE ${SCORPIO_C_LIB} gptl ${netcdf_c_lib}) - target_include_directories (pioc INTERFACE ${SCORPIO_INC_DIR} ${netcdf_c_incdir} ${pnetcdf_incdir} ${CSM_SHR_INCLUDE}) + target_link_libraries (pioc INTERFACE spio) + add_library (piof INTERFACE) + target_link_libraries (piof INTERFACE spio) + + #set(SCORPIO_INC_DIR ${INSTALL_SHAREDPATH}/include) # HACK: CIME only copies headers from the bld dir to the CSM_SHR_INCLUDE dir # This means all the pioc headers in the src folder are not copied. # It would be nice if CIME used the cmake-generated makefile, and # ran 'make install' rather than copy files. Alas, we don't control # that, so we need another way. Including the src tree folder works. - target_include_directories (pioc INTERFACE ${SCREAM_BASE_DIR}/../../externals/scorpio/src/clib) - get_target_property (pioc_inc_dirs pioc INTERFACE_INCLUDE_DIRECTORIES) - message ("pioc includes: ${pioc_inc_dirs}") - if (pnetcdf_lib) - target_link_libraries(pioc INTERFACE "${pnetcdf_lib}") - endif () - - ###################### - # PIOf # - ###################### - - # Look for piof lib in INSTALL_SHAREDPATH/lib - find_library(SCORPIO_F_LIB piof REQUIRED PATHS ${SCORPIO_LIB_DIR}) - - # Create the interface library, and set target properties - add_library(piof INTERFACE) - target_link_libraries (piof INTERFACE ${SCORPIO_F_LIB} ${netcdf_f_lib} pioc) - target_include_directories (piof INTERFACE ${SCORPIO_INC_DIR} ${netcdf_f_incdir} ) + target_include_directories(pioc INTERFACE ${SCREAM_BASE_DIR}/../../externals/scorpio/src/clib) else () # Not a CIME build. We'll add scorpio as a subdir diff --git a/components/eamxx/data/scream_default_output.yaml b/components/eamxx/data/scream_default_output.yaml index 81ccf0293dac..7e0a45f12d6a 100644 --- a/components/eamxx/data/scream_default_output.yaml +++ b/components/eamxx/data/scream_default_output.yaml @@ -35,8 +35,10 @@ Fields: - qr - eff_radius_qc - eff_radius_qi + - eff_radius_qr - precip_ice_surf_mass - precip_liq_surf_mass + - rainfrac # SHOC + P3 - qc - qv @@ -59,6 +61,13 @@ Fields: - surf_sens_flux # Diagnostics - PotentialTemperature + # GLL output for homme states. + Dynamics: + Field Names: + - ps_dyn + - dp3d_dyn + - omega_dyn + IO Grid Name: Physics GLL output_control: # WARNING: ERS/ERP tets will override this with STOP_N/STOP_OPTION Frequency: ${HIST_N} diff --git a/components/eamxx/data/scream_default_remap.yaml b/components/eamxx/data/scream_default_remap.yaml index 6749034adc4b..8bf47386c76d 100644 --- a/components/eamxx/data/scream_default_remap.yaml +++ b/components/eamxx/data/scream_default_remap.yaml @@ -35,8 +35,10 @@ Fields: - qr - eff_radius_qc - eff_radius_qi + - eff_radius_qr - precip_ice_surf_mass - precip_liq_surf_mass + - rainfrac # SHOC + P3 - qc - qv diff --git a/components/eamxx/docs/README.md b/components/eamxx/docs/README.md deleted file mode 100644 index 5c6e4a1de213..000000000000 --- a/components/eamxx/docs/README.md +++ /dev/null @@ -1,15 +0,0 @@ -This is the top-level of the SCREAM design documentation. That documentation -is written in LaTeX and is spread over a directory structure mimicking that -of the actual code within the ../src directory. - -To compile the documentation for a particular process, go to the docs -directory for that process and issue: - -pdflatex main.tex -bibtex main -pdflatex main.tex - -If you want to compile the documentation for the whole model, issue the same -command from the top-level docs directory. - -Obviously, you need to have a working Latex installation for this to work. \ No newline at end of file diff --git a/components/eamxx/docs/build.md b/components/eamxx/docs/build.md deleted file mode 100644 index 7ca9f05e1a93..000000000000 --- a/components/eamxx/docs/build.md +++ /dev/null @@ -1,162 +0,0 @@ -# Building and Testing SCREAM Unit Tests - -Follow these simple instructions to build and test SCREAM's standalone -configuration for yourself. Note that similar documentation is available on confluence (for E3SM team members) -at https://acme-climate.atlassian.net/wiki/spaces/NGDNA/pages/1264386127/Running+SCREAM+Tests. -This document makes use of the following paths: - -+ `${RUN_ROOT_DIR}`: the root directory where SCREAM is built and run -+ `${SCREAM_SRC_DIR}`: the directory into which you've cloned the `scream` repo - -SCREAM's configuration and build system is based on [CMake](https://cmake.org/). -CMake has been around a while and has gained a lot of traction in recent years, -especially in the HPC community. It has good [reference documentation](https://cmake.org/cmake/help/latest/index.html), -but it can be tricky to use if you've never encountered it. Ask a SCREAM team -member for help if you're stuck on something CMake-related. - -If you see a `CMakeLists.txt` files or a file with a `.cmake` suffix, that's -just part of the build system. You might also see files with `CTest` as part of -their name. These files are related to [CTest](https://cmake.org/cmake/help/latest/manual/ctest.1.html), -CMake's testing tool. - -## 1. Start From a Trustworthy Commit - -First, make sure you've cloned the [SCREAM repo (including all submodules)](https://github.com/E3SM-Project/scream) -to `SCREAM_SRC_DIR` using the following command: - -``` -git clone --recurse-submodules https://github.com/E3SM-Project/scream -``` - -If you have already cloned the project and forgot to type `--recurse-submodules`, -you can change to `$SCREAM_SRC_DIR` and using the following command to initialize, -fetch and checkout all submodules: - -``` -git submodule update --init --recursive -``` - -If you're running a branch that's not `master`, check out this branch with - -``` -git checkout -``` - -## 2. Configure Your SCREAM Build - -Change to your `$RUN_ROOT_DIR` directory and use CMake to configure your build. - -If you're building SCREAM on one of our supported platforms, you can tell CMake -to use the appropriate machine file using the `-C` flag. Machine files are -located in `$SCREAM_SRC_DIR/components/eamxx/cmake/machine-files`. Take a look -and see whether your favorite machine has one. - -For example, to configure SCREAM on the Quartz machine at LLNL: - -``` -cd $RUN_ROOT_DIR -cmake \ - -DCMAKE_CXX_COMPILER=$(which mpicxx) \ - -DCMAKE_BUILD_TYPE=Debug \ - -C ${SCREAM_SRC_DIR}/components/eamxx/cmake/machine-files/quartz.cmake \ - ${SCREAM_SRC_DIR}/components/eamxx -``` - -If you're building on a machine that doesn't have a ready-made machine file, -you can try configuring your build by manually passing options to CMake. This -usually looks something like the following: -``` -cd $RUN_ROOT_DIR -cmake \ - -D CMAKE_BUILD_TYPE=Debug \ - -D Kokkos_ENABLE_DEBUG=TRUE \ - -D Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION=OFF \ - -D Kokkos_ENABLE_SERIAL=ON \ - -D Kokkos_ENABLE_OPENMP=ON \ - -D Kokkos_ENABLE_PROFILING=OFF \ - -D Kokkos_ENABLE_DEPRECATED_CODE=OFF \ - -D KOKKOS_ENABLE_ETI:BOOL=OFF \ - -D CMAKE_C_COMPILER=mpicc \ - -D CMAKE_CXX_COMPILER=mpicxx \ - -D CMAKE_Fortran_COMPILER=mpif90 \ - ${SCREAM_SRC_DIR}/components/eamxx -``` - -In either case, SCREAM requires MPI-savvy compilers, which can be specified -using the `CMAKE_xyz_COMPІLER` options. - -Above, we've configured `Debug` builds to make it easier to find and fix errors. -For performance testing, you should configure a `Release` build and make use of -other options, depending on your architecture. - -## 3. Build SCREAM - -Now you can build SCREAM from that same directory: - -``` -make -j -``` - -The `-j` flag tells Make to use threads to compile in parallel. If you like, you -can set the number of threads by passing it as an argument to `-j` (e.g. -`make -j8`). - -## 4. Run SCREAM's Tests - -Before running the tests, generate a baseline file: - -``` -cd $RUN_ROOT_DIR -make baseline -``` - -The tests will run, automatically using the baseline file, which is located in -the CMake-configurable path `${SCREAM_TEST_DATA_DIR}`. By default, this path is -set to `data/` within your build directory (which is `$RUN_ROOT_DIR`, in -our case). - -To run all of SCREAM's tests, make sure you're in `$RUN_ROOT_DIR` and type - -``` -ctest -VV -``` - -This runs everything and reports results in an extra-verbose (`-VV`) manner. - -You can also run subsets of the SCREAM tests. For example, to run only the -P3 regression tests (again, from the `$RUN_ROOT_DIR` directory), use - -``` -ctest -R p3_regression -``` - -### Grouping Tests with Labels - -We can create groupings of tests by using **labels**. For example, we have a -`driver` label that runs tests for SCREAM's standalone driver. You can see a -list of available labels by typing - -``` -ctest --print-labels -``` - -To see which tests are associated with a given label (e.g. `driver`), use - -``` -ctest -L driver -N -``` - -# SCREAM Test Suites - -## The `p3_regression` Suite - -`p3_regression` uses a baseline file to compare any new or altered -implementations with our P3 Fortran reference implementation. If you're working -on the C++/Kokkos implementation, you can invoke any new tests to the function -`Baseline::run_and_cmp` in -`${SCREAM_SRC_DIR}/components/eamxx/p3/tests/p3_run_and_cmp.cpp`. - -If the reference Fortran implementation changes enough that a new baseline file -is required, make sure to let other SCREAM team members know, in order to -minimize disruptions. - diff --git a/components/eamxx/docs/common/installation.md b/components/eamxx/docs/common/installation.md new file mode 100644 index 000000000000..017007abdce8 --- /dev/null +++ b/components/eamxx/docs/common/installation.md @@ -0,0 +1,170 @@ +# Installation + +Follow these simple instructions to build and test EAMxx's standalone +configuration for yourself. This document makes use of the following paths: + ++ `${RUN_ROOT_DIR}`: the root directory where EAMxx is built and run ++ `${EAMXX_SRC_DIR}`: the directory into which you've cloned the `scream` repo + +EAMxx's configuration and build system is based on [CMake](https://cmake.org/). +CMake has been around a while and has gained a lot of traction in recent years, +especially in the HPC community. It has good [reference documentation](https://cmake.org/cmake/help/latest/index.html), +but it can be tricky to use if you've never encountered it. Ask a EAMxx team +member for help if you're stuck on something CMake-related. + +If you see a `CMakeLists.txt` files or a file with a `.cmake` suffix, that's +just part of the build system. You might also see files with `CTest` as part of +their name. These files are related to [CTest](https://cmake.org/cmake/help/latest/manual/ctest.1.html), +CMake's testing tool. + +## Prerequisites + +First, make sure you're on one of the machines supported by EAMxx, or that you +have the following software installed: + +* A working MPI installation (typically [MPICH]() or [Open-MPI]()) +* [CMake](https://cmake.org) and [GNU Make](https://www.gnu.org/software/make/) +* A working set of C, C++, and Fortran compilers +* A recent version of [Git](https://git-scm.com/) +* A working installation of [NetCDF](https://www.unidata.ucar.edu/software/netcdf/), + including both [C](https://github.com/Unidata/netcdf-c) and + [Fortran](https://github.com/Unidata/netcdf-fortran) libraries. + +## Setting Up Your Environment + +## Configuring and Building Scream + +### 1. Start From a Trustworthy Commit + +First, make sure you've cloned the [EAMxx repo (including all submodules)](https://github.com/E3SM-Project/scream) +to `EAMXX_SRC_DIR` using the following command: + +``` +git clone --recurse-submodules https://github.com/E3SM-Project/scream +``` + +If you have already cloned the project and forgot to type `--recurse-submodules`, +you can change to `$EAMXX_SRC_DIR` and using the following command to initialize, +fetch and checkout all submodules: + +``` +git submodule update --init --recursive +``` + +If you're running a branch that's not `master`, check out this branch with + +``` +git checkout +``` + +### 2. Configure Your EAMxx Build + +Change to your `$RUN_ROOT_DIR` directory and use CMake to configure your build. + +If you're building SCREAM on one of our supported platforms, you can tell CMake +to use the appropriate machine file using the `-C` flag. Machine files are +located in `$EAMXX_SRC_DIR/components/eamxx/cmake/machine-files`. Take a look +and see whether your favorite machine has one. + +For example, to configure SCREAM on the Quartz machine at LLNL: + +``` +cd $RUN_ROOT_DIR +cmake \ + -DCMAKE_CXX_COMPILER=$(which mpicxx) \ + -DCMAKE_BUILD_TYPE=Debug \ + -C ${EAMXX_SRC_DIR}/components/eamxx/cmake/machine-files/quartz.cmake \ + ${EAMXX_SRC_DIR}/components/eamxx +``` + +If you're building on a machine that doesn't have a ready-made machine file, +you can try configuring your build by manually passing options to CMake. This +usually looks something like the following, which configures EAMxx to compile +CPU code using Kokkos's OpenMP backend: +``` +cd $RUN_ROOT_DIR +cmake \ + -D CMAKE_BUILD_TYPE=Debug \ + -D CMAKE_C_COMPILER=mpicc \ + -D CMAKE_CXX_COMPILER=mpicxx \ + -D CMAKE_Fortran_COMPILER=mpif90 \ + -D MPIEXEC_EXECUTABLE=`which mpiexec` \ + -D EKAT_MPI_NP_FLAG:STRING=-n \ + -D SCREAM_DYNAMICS_DYCORE=HOMME \ + -D SCREAM_DOUBLE_PRECISION:BOOL=ON \ + -D SCREAM_INPUT_ROOT:PATH=/path/to/scream-input \ + -D Kokkos_ENABLE_DEBUG=TRUE \ + -D Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION=OFF \ + -D Kokkos_ENABLE_SERIAL=ON \ + -D Kokkos_ENABLE_OPENMP=ON \ + -D Kokkos_ENABLE_LIBDL=OFF \ + -D Kokkos_ENABLE_PROFILING=OFF \ + -D Kokkos_ENABLE_DEPRECATED_CODE=OFF \ + -D KOKKOS_ENABLE_ETI:BOOL=OFF \ + -D NetCDF_C_PATHS=/path/to/netcdf-c-dir \ + -D NetCDF_Fortran_PATHS=/path/to/netcdf-f90-dir \ + -D PnetCDF_C_PATHS=/path/to/pnetcdf-dir \ + -D PnetCDF_Fortran_PATHS=/path/to/pnetcdf-f90-dir \ + ${EAMXX_SRC_DIR}/components/eamxx +``` + +In either case, EAMxx requires MPI-aware compilers. Let's examine these +options (only some of which are required on any given machine) to make sure we +know what they do: + +* `CMAKE_BUILD_TYPE`: specifies whether you are building EAMxx in a + developer-friendly configuration (`Debug`), for a production run (`Release`) + or for performance profiling or some other specialized purpose. Typically, + you'll set this option to `Debug` or `Release`. +* `CMAKE_{C,CXX,Fortran}_COMPILER`: the name of the command used to invoke an + MPI-enabled C, C++, or Fortran compiler to build EAMxx +* `MPIEXEC_EXECUTABLE`: the name of the command used to run EAMxx using MPI, + typically `mpiexec` or `mpirun`, but possibly different depending on your + desired machine +* `EKAT_MPI_NP_FLAG`: the flag passed to `MPIEXEC_EXECUTABLE` that you use to + specify the number of desired MPI processes. This is typically `-n` for + `mpiexec` and `-np` for `mpirun`. +* `SCREAM_DYNAMICS_DYCORE`: specifies the dycore used for configuring EAMxx, + which is `NONE` if you are not configuring EAMxx to run its dycore-related + tests, or `HOMME` if you want to use HOMMExx +* `SCREAM_DOUBLE_PRECISION`: indicates whether EAMxx's `Real` type is a + double-precision (`ON`) or single-precision (`OFF`) floating point type +* `SCREAM_INPUT_ROOT`: specifies the location of the top-level folder that + stores input data files for EAMxx. This folder is populated with input files + which are downloaded automatically during EAMxx's build process. +* The Kokkos-related build options (most of which begin with `Kokkos_`) are + described [in the Kokkos Wiki](https://kokkos.github.io/kokkos-core-wiki/keywords.html) +* `NetCDF_C_PATHS`: specifies one or more folders in which the NetCDF C library + and headers are installed. In the simplest configuration, the headers should + be located in `${NetCDF_C_PATHS}/include` and the library should live in + `${NetCDF_C_PATHS}/lib`. +* `NetCDF_Fortran_PATHS`: specifies one or more folders in which the NetCDF + Fortran library and modules are installed. Analogous to `${NetCDF_C_PATHS}`, + `.mod` files should be in `${NetCDF_Fortran_PATHS}/include`, and the library + should be installed in `${NetCDF_Fortran_PATHS}/lib`. +* `PnetCDF_C_PATHS`: specifies one or more folders in which the pNetCDF C + library and headers are installed, analogous to `NetCDF_C_PATHS`. +* `PnetCDF_Fortran_PATHS`: specifies one or more folders in which the pNetCDF + Fortran library and modules are installed, analogous to + `NetCDF_Fortran_PATHS`. + +Above, we've configured `Debug` builds to make it easier to find and fix errors. +For performance testing, you should configure a `Release` build and make use of +other options, depending on your architecture. + +### 3. Build SCREAM + +Now you can build SCREAM from that same directory: + +``` +make -j +``` + +The `-j` flag tells Make to use threads to compile in parallel. If you like, you +can set the number of threads by passing it as an argument to `-j` (e.g. +`make -j8`). + +## Running Tests + +You can run EAMxx's tests to make sure your build works by following the +instructions [here](../developer/standalone_testing.md). diff --git a/components/eamxx/docs/control/main.tex b/components/eamxx/docs/control/main.tex deleted file mode 100644 index 4ce1ffab4733..000000000000 --- a/components/eamxx/docs/control/main.tex +++ /dev/null @@ -1,35 +0,0 @@ -\documentclass[12pt]{article} -\usepackage{authblk} %needed to compile chunks as standalone -\bibliographystyle{../amermeteorsoc} - -\title{A next generation driver for EAMxx} - -\author[2]{Luca Bertagna} -\author[1]{Aaron Donahue} -\author[2]{Ben Hillman} -\author[1]{Peter Caldwell} -\author[2]{Thomas Clevenger} -\author[2]{Jim Foucar} -\date{\today} - -\affil[1]{Lawrence Livermore National Lab, Livermore CA} -\affil[2]{Sandia National Laboratories, Albuquerque, NM} -\affil[3]{Lawrence Berkeley National Laboratory, Berkeley, CA} -\affil[4]{Brookhaven National Laboratory, Upton, NY} -\affil[5]{Pacific Northwest National Laboratory, Richland, WA} -\affil[6]{University of California, Davis, Davis, CA} - -\begin{document} -\maketitle{} - - -\input{driver_doc.tex} - -%bibliography needs to be in main b/c will be called from different directories when an individual -%section is compiled versus when all documentation is compiled. -%================================ -\subsection{Bibliography} -%================================ -\bibliography{../bibliography.bib} - -\end{document} diff --git a/components/eamxx/docs/developer/ci_nightly.md b/components/eamxx/docs/developer/ci_nightly.md new file mode 100644 index 000000000000..4089f523daf9 --- /dev/null +++ b/components/eamxx/docs/developer/ci_nightly.md @@ -0,0 +1,4 @@ +# Continuous Integration and Nightly Testing + +* Autotester quick overview +* Nightly overview, CDash diff --git a/components/eamxx/docs/developer/cime_testing.md b/components/eamxx/docs/developer/cime_testing.md new file mode 100644 index 000000000000..7c2d91a36579 --- /dev/null +++ b/components/eamxx/docs/developer/cime_testing.md @@ -0,0 +1,7 @@ +# Full Model Testing + +Quickly review CIME test infrastructure and how EAMxx uses it + +* test types, specifiers (`_LnX`,`_D`,`_PMxN`,..), grids, compsets, test-mods +* available grids/compsets for EAMxx, and where to find them +* how to add atmchange in `shell_commands` test mods diff --git a/components/eamxx/docs/developer/field.md b/components/eamxx/docs/developer/field.md new file mode 100644 index 000000000000..013dcf857b62 --- /dev/null +++ b/components/eamxx/docs/developer/field.md @@ -0,0 +1,46 @@ +## Field + +In EAMxx, a `Field` is a data structure holding two things: pointers to the data and pointers to metadata. +Both the data and metadata are stored in `std::shared_ptr` instances, to ensure consistency across all copies +of the field. This allows for fast shallow copy semantic for this class. + +The data is stored on both CPU and device memory (these may be the same, depending on the Kokkos +backend). In EAMxx, we always assume and guarantee that the device data is up to date. That implies that the data +be explicitly synced to host before using it on host, and explicitly synced to device after host manipulation, +in order to ensure correctness. In order to access the data, users must use the `get_view` method, which takes +two template arguments: the data type, and an enum specifying whether CPU or device data is needed. The data +type is used to reinterpret the generic pointer stored inside to a view of the correct scalar type and layout. +It is a possibly const-qualified type, and if the field was marked as "read-only", the method ensures that the +provided data type is const. A read-only field can be created via the `getConst` method, which returns an +identical copy of the field, but marked as read-only. The enum specifying host or device data is optional, +with device being the default. + +The metadata is a collection of information on the field, such as name, layout, units, allocation size, and more. +Part of the metadata is immutable after creation (e.g., name, units, or layout), while some metadata can be +partially or completely modified. The metadata is contained in the `FieldHeader` data structure, which contains +four parts: + +* `FieldIdentifier`: stores the field's name, layout, units, data type, and name of the grid where it's defined. + These information are condensed in a single string, that can be used to uniquely identify a field, + allowing to distinguish between different version of the same field. The layout is stored in the `FieldLayout` + data structure, which includes: + * the field tags: stored as a `std::vector`, they give context to the field's extents. + * the field dims: stored both as a `std::vector`, as well as a 1d `Kokkos::View`. +* `FieldTracking`: stores information on the usage of the field, as well as its possible connections to other + fields. In particular, the tracked items are: + * the field time stamp: the time stamp when the field was last updated. + * the field accumulation start time: used for fields that are accumulated over several time steps + (or time step subcycles). For instance, it allows to reconstruct fluxes from raw accumulations. + * the providers/customers: lists of atmosphere processes (see below) that respectively require/compute + the field in their calculations. + * the field groups: a list of field groups that this field belongs too. Field groups are used to access + a group of fields without explicit prior knowledge about the number and/or names of the fields. +* `FieldAllocProp`: stores information about the allocation. While the field is not yet allocated, users can + request special allocations for the field, for instance to accommodate packing (for SIMD), which may + require padding. Upon allocation, this information is then used by the Field structure to extract the + actual data, wrapped in a properly shaped `Kokkos::View`. The alloc props are also responsible of tracking + additional information in case the field is a "slice" of a higher-dimensional one, a fact that can affect + how the data is accessed. +* Extra data: stored as a `std::map`, allows to catch any metadata that does not fit + in the above structures. This is a last resort structure, intended to accommodate the most peculiar + corner cases, and should be used sparingly. diff --git a/components/eamxx/docs/developer/grid.md b/components/eamxx/docs/developer/grid.md new file mode 100644 index 000000000000..8a61b97e0795 --- /dev/null +++ b/components/eamxx/docs/developer/grid.md @@ -0,0 +1,22 @@ +## Grids and Remappers + +In EAMxx, the `AbstractGrid` is an interface used to access information regarding the horizontal and vertical +discretization. The most important information that the grid stores is: + +* the number of local/global DOFs: these are the degrees of freedom of the horizontal grid only. Here, + local/global refers to the MPI partitioning. +* the DOFs global IDs (GIDs): a list of GIDs of the DOFs on the current MPI rank, stored as a Field +* the local IDs (LIDs) to index list: this list maps the LID of a DOF (that is, the position of the DOF + in the GID list) to a "native" indexing system for that DOF. For instance, a `PointGrid` (a class derived from + `AbstractGrid`) is a simple collection of points, so the "native" indexing system coincides with the LIDs. + However, for a `SEGrid` (a derived class, for spectral element grids), the "native" indexing is a triplet + `(ielem,igp,jgp)`, specifying the element index, and the two indices of the Gauss point within the element. +* geometry data: stored as a `std::map`, this represent any data that is intrinsically + linked to the grid (either along the horizontal or vertical direction), such as lat/lon coordinates, + vertical coordinates, area associated with the DOF. + +Grids can also be used to retrieve the layout of a 2d/3d scalar/vector field, which allows certain downstream +classes to perform certain operations without assuming anything on the horizontal grid. + +In general, grid objects are passed around the different parts of EAMxx as const objects (read-only). +The internal data can only be modified during construction, which usually is handled by a `GridsManager` object. diff --git a/components/eamxx/docs/developer/index.md b/components/eamxx/docs/developer/index.md new file mode 100644 index 000000000000..2d47bab65fe3 --- /dev/null +++ b/components/eamxx/docs/developer/index.md @@ -0,0 +1,3 @@ +# SCREAM Developer Guide + + diff --git a/components/eamxx/docs/developer/io.md b/components/eamxx/docs/developer/io.md new file mode 100644 index 000000000000..caf237010a33 --- /dev/null +++ b/components/eamxx/docs/developer/io.md @@ -0,0 +1,5 @@ +# Input-Output + +In EAMxx, I/O is handled through the SCORPIO library, currently a submodule of E3SM. +The `scream_io` library within eamxx allows to interface the EAMxx infrastructure classes +with the SCORPIO library. diff --git a/components/eamxx/docs/developer/kokkos_ekat.md b/components/eamxx/docs/developer/kokkos_ekat.md new file mode 100644 index 000000000000..4a5df20ab80a --- /dev/null +++ b/components/eamxx/docs/developer/kokkos_ekat.md @@ -0,0 +1,12 @@ +# Building Blocks + +Here we can discuss EKAT, Kokkos, and all of the highly-technical non-scientific +stuff that makes our heads hurt. + +## Kokkos Views + +## Vectorization: Packs + +## Fields and the Field Manager + +### Preconditions, Postconditions, and Invariants diff --git a/components/eamxx/docs/developer/managers.md b/components/eamxx/docs/developer/managers.md new file mode 100644 index 000000000000..676449a21845 --- /dev/null +++ b/components/eamxx/docs/developer/managers.md @@ -0,0 +1 @@ +## FieldManager and GridsManager diff --git a/components/eamxx/docs/developer/processes.md b/components/eamxx/docs/developer/processes.md new file mode 100644 index 000000000000..d6a81b3cae20 --- /dev/null +++ b/components/eamxx/docs/developer/processes.md @@ -0,0 +1,18 @@ +# Atmospheric Processes + +In EAMxx, the `AtmosphereProcess` (AP) is a class representing a portion of the atmosphere timestep algorithm. +In simple terms, an AP is an object that given certain input fields performs some calculations to compute +some output fields. + +TODO: describe init sequcene (e.g., the process of requesting fields), base class main + interfaces/capabilities (e.g., subcycling), class expectations (e.g., must update fields on physics grid) + +Here is a list of currently implemented atmosphere processes. +TODO: add links to papers/github-repos, and a SMALL description +* p3: Microphysics, blah blah +* SHOC: Macrophysics/Turbulence, blah +* rrtmgp: Radiation, blah +* spa: prescribed aerosols, blah blah +* surface coupling: blah +* mam: prognostic aerosols, blah blah +* nudging: This process is responsible for nudging the model simulation given a set of files with a target nudged state. diff --git a/components/eamxx/docs/developer/source_tree.md b/components/eamxx/docs/developer/source_tree.md new file mode 100644 index 000000000000..15c018cc8858 --- /dev/null +++ b/components/eamxx/docs/developer/source_tree.md @@ -0,0 +1,59 @@ +# EAMxx's Source Tree + +All EAMxx-specific code can be found in `components/eamxx` within the +[EAMxx repo](https://github.com/E3SM-Project/scream). Here's how things are +organized: + ++ `cime_config`: Tools and XML files for integrating EAMxx with E3SM via the + CIME framework. ++ `cmake`: CMake functions and macros used by the configuration/build system. ++ `data`: Data files used by our tests. ++ `docs`: Documentation for the EAMxx project, including design documents, + instructions for building and testing EAMxx, and this document. ++ `scripts`: Miscellaneous scripts that implement workflows for running tests + and analyzing performance. ++ `src`: All C++ source code (and any bridges to Fortran) for EAMxx are stored + here. We describe the contents of this directory in greater detail below. ++ `tests`: Implements standalone, end-to-end tests for various EAMxx + components (RRTMG, HOMME, P3, SHOC, etc). + +In addition, you'll notice the following files in `components/eamxx`: + ++ `CMakeLists.txt`: The CMake file that defines EAMxx's configuration/build + system. ++ `CTestConfig.cmake`: This CTest file contains parameters that determine how + our test results are reported to the [E3SM CDash Site](http://my.cdash.org/submit.php?project=E3SM). ++ `README.md`: EAMxx's top-level README file, which describes the project and + its purpose. ++ `mkdocs.yml`: The configuration file for [mkdocs](https://www.mkdocs.org/), + the tool we currently use to build and publish our documentation. + +## The `src` Directory + +Herein lіes the source code for EAMxx. Broadly, here's where things are: + ++ `control`: Contains the atmosphere driver and basic tests for it. ++ `dynamics`: Here's where HOMME lives within EAMxx, along with code for + interfacing with it using EAMxx's data structures. ++ `mct_coupling`: Glue code for embedding EAMxx within E3SM as an atmosphere + component using the MCT coupler. ++ `physics`: Source code for physics-related atmospheric processes, including + + `p3`: The C++/Kokkos implementation of P3 microphysics within EAMxx. + + `shoc`: The C++/Kokkos implementation of SHOC macrophysics within EAMxx. + + `rrtmgp`: A stub for the radiation processes as represented in EAMxx. + + `share`: Utilities and data structures common to these processes. ++ `share`: Utilities used by various components within EAMxx. Of note: + + `io`: EAMxx's interface to the [SCORPIO](https://e3sm.org/scorpio-parallel-io-library/) + library. ++ `diagnostics`: A collection of simple classes used to compute diagnostic + quantities. + +Each of these directories contains a `CMakeLists.txt` file for defining how +things are build, and a `tests/` subdirectory that houses relevant +unit and verification tests. + +You'll also see some other files in the `src/` directory itself, such as + ++ `scream_config.h.in`: A template for generating a C++ header file with + EAMxx configuration information. + diff --git a/components/eamxx/docs/developer/standalone_testing.md b/components/eamxx/docs/developer/standalone_testing.md new file mode 100644 index 000000000000..aedb7b2cbaad --- /dev/null +++ b/components/eamxx/docs/developer/standalone_testing.md @@ -0,0 +1,84 @@ +# Standalone EAMxx Testing + +In this section we describe our testing methodology for standalone EAMxx +configurations. We use several types of tests + +* **Unit tests** are individual test programs that demonstrate that a small set + of code performs a single function or a set of related functions. We use + a C++ unit testing framework called [Catch2](https://catch2-temp.readthedocs.io/en/latest/index.html) + to implement unit tests. +* **Property (verification) tests** are test programs that configure code that + demonstrates that a part of EAMxx (for example, an atmospheric physics + parameterization or the dynamical core) is able to produce an answer that + satisfies some physical constraint or matches a known solution under specific + circumstances. +* **Fortran-C++ "bit-for-bit" (BFB) tests** are test programs, often implemented + as unit tests, that demonstrate that a set of C++ code ported from Fortran + produces bit-for-bit identical results to its Fortran counterpart, provided + certain compiler options are enabled (such as "strict" floating-point + arithmetic). +* **Test Suites** are named collections of tests that can be run on demand using + the [ctest](https://cmake.org/cmake/help/latest/manual/ctest.1.html) command. + +We also support a `test-all-scream` configuration that runs all of the +standalone tests for an EAMxx configuration. + +## Running EAMxx's Tests with CTest + +Before running the tests, generate a baseline file: + +``` +cd $RUN_ROOT_DIR +make baseline +``` + +The tests will run, automatically using the baseline file, which is located in +the CMake-configurable path `${SCREAM_TEST_DATA_DIR}`. By default, this path is +set to `data/` within your build directory (which is `$RUN_ROOT_DIR`, in +our case). + +To run all of SCREAM's tests, make sure you're in `$RUN_ROOT_DIR` and type + +``` +ctest -VV +``` + +This runs everything and reports results in an extra-verbose (`-VV`) manner. + +You can also run subsets of the SCREAM tests. For example, to run only the +P3 regression tests (again, from the `$RUN_ROOT_DIR` directory), use + +``` +ctest -R p3_regression +``` + +### Grouping Tests with Labels + +We can create groupings of tests by using **labels**. For example, we have a +`driver` label that runs tests for SCREAM's standalone driver. You can see a +list of available labels by typing + +``` +ctest --print-labels +``` + +To see which tests are associated with a given label (e.g. `driver`), use + +``` +ctest -L driver -N +``` + +## EAMxx Test Suites + +### The `p3_regression` Suite + +`p3_regression` uses a baseline file to compare any new or altered +implementations with our P3 Fortran reference implementation. If you're working +on the C++/Kokkos implementation, you can invoke any new tests to the function +`Baseline::run_and_cmp` in +`${SCREAM_SRC_DIR}/components/eamxx/p3/tests/p3_run_and_cmp.cpp`. + +If the reference Fortran implementation changes enough that a new baseline file +is required, make sure to let other SCREAM team members know, in order to +minimize disruptions. + diff --git a/components/eamxx/docs/developer/style_guide.md b/components/eamxx/docs/developer/style_guide.md new file mode 100644 index 000000000000..f43678330099 --- /dev/null +++ b/components/eamxx/docs/developer/style_guide.md @@ -0,0 +1,10 @@ +# SCREAM C++ Style Guide + +Here's our style guide. Let the holy wars begin! + +## Types + +## Functions and Methods + +## Variables + diff --git a/components/eamxx/docs/dynamics/.DS_Store b/components/eamxx/docs/dynamics/.DS_Store deleted file mode 100644 index 243ceaf8ed8a..000000000000 Binary files a/components/eamxx/docs/dynamics/.DS_Store and /dev/null differ diff --git a/components/eamxx/docs/dynamics/homme/NHxx_doc.tex b/components/eamxx/docs/dynamics/homme/NHxx_doc.tex deleted file mode 100644 index 0b616743913b..000000000000 --- a/components/eamxx/docs/dynamics/homme/NHxx_doc.tex +++ /dev/null @@ -1,17 +0,0 @@ -\section{Nonhydrostatic Spectral Element Dycore} - -\subsection{Theory} - -Put explanation of continuous equations here. - -\subsection{Numerical Methods} - -Describe the numerical methods used to discretize the equations here. - -\subsection{Computational Implementation} - -Describe the strategies used (if any) to ensure good computational performance. - -\subsection{Verification} - -Describe testing strategy diff --git a/components/eamxx/docs/dynamics/homme/main.tex b/components/eamxx/docs/dynamics/homme/main.tex deleted file mode 100644 index 00e6f2d75e7d..000000000000 --- a/components/eamxx/docs/dynamics/homme/main.tex +++ /dev/null @@ -1,41 +0,0 @@ -\documentclass[12pt]{article} -\usepackage{authblk} - -\title{Design Document for the Non-hydrostatic Spectral-Element Dycore Used by SCREAM} - -\author[1]{Peter Caldwell} -\author[2]{Andy Salinger} -\author[2]{Luca Bertagna} -\author[1]{Hassan Beydoun} -\author[1]{Peter Bogenschutz} -\author[2]{Andrew Bradley} -\author[1]{Aaron Donahue} -\author[2]{Jim Foucar} -\author[1]{Chris Golaz} -\author[2]{Oksana Guba} -\author[2]{Ben Hillman} -\author[3]{Noel Keen} -\author[4]{Wuyin Lin} -\author[5]{Kyle Pressel} -\author[5]{Balwinder Singh} -\author[2]{Andrew Steyer} -\author[2]{Mark Taylor} -\author[1]{Chris Terai} -\author[6]{Paul Ullrich} -\date{\today} - -\affil[1]{Lawrence Livermore National Lab, Livermore CA} -\affil[2]{Sandia National Laboratories, Albuquerque, NM} -\affil[3]{Lawrence Berkeley National Laboratory, Berkeley, CA} -\affil[4]{Brookhaven National Laboratory, Upton, NY} -\affil[5]{Pacific Northwest National Laboratory, Richland, WA} -\affil[6]{University of California, Davis, Davis, CA} - -\begin{document} -\maketitle{} - -NOTE: author list for just this section should be culled to just the people working on this section. - -\input{NHxx_doc.tex} - -\end{document} diff --git a/components/eamxx/docs/index.md b/components/eamxx/docs/index.md new file mode 100644 index 000000000000..992b797671cf --- /dev/null +++ b/components/eamxx/docs/index.md @@ -0,0 +1,10 @@ +# The C++ E3SM Atmosphere Model (EAMxx) + +Some nice introductory text goes here! Maybe some figures, too. Who knows?! + +* The [User Guide](user/index.md) explains how to run EAMxx, both in + its standalone configuration and within E3SM. +* The [Developer Guide](developer/index.md) contains all the information needed + to contribute to the development of EAMxx. +* The [Technical Guide](technical/index.md) contains all the technical + information about EAMxx. diff --git a/components/eamxx/docs/main.tex b/components/eamxx/docs/main.tex deleted file mode 100644 index b7969bd723aa..000000000000 --- a/components/eamxx/docs/main.tex +++ /dev/null @@ -1,67 +0,0 @@ -\documentclass[12pt]{article} -\usepackage{authblk} -\usepackage{graphicx} %used by PSL at least -\usepackage{amsmath} %used by SHOC at least -\usepackage{natbib} %allows use of \citep{} and \citet{} -\bibliographystyle{amermeteorsoc} - -\makeindex - -\title{Design Document for the Simple Cloud-Resolving E3SM Atmosphere Model} -\author[1]{Peter Caldwell} -\author[2]{Andy Salinger} -\author[2]{Luca Bertagna} -\author[1]{Hassan Beydoun} -\author[1]{Peter Bogenschutz} -\author[2]{Andrew Bradley} -\author[5]{Conrad Clevenger} -\author[1]{Aaron Donahue} -\author[2]{Jim Foucar} -\author[1]{Chris Golaz} -\author[2]{Oksana Guba} -\author[2]{Ben Hillman} -\author[3]{Noel Keen} -\author[4]{Wuyin Lin} -\author[5]{Balwinder Singh} -\author[2]{Andrew Steyer} -\author[2]{Mark Taylor} -\author[1]{Chris Terai} -\author[6]{Paul Ullrich} -\date{\today} - -\affil[1]{Lawrence Livermore National Lab, Livermore CA} -\affil[2]{Sandia National Laboratories, Albuquerque, NM} -\affil[3]{Lawrence Berkeley National Laboratory, Berkeley, CA} -\affil[4]{Brookhaven National Laboratory, Upton, NY} -\affil[5]{Pacific Northwest National Laboratory, Richland, WA} -\affil[6]{University of California, Davis, Davis, CA} - -\begin{document} -\maketitle{} - -\setcounter{tocdepth}{4} -\setcounter{secnumdepth}{4} -\tableofcontents - -\newpage - -\section{Overview} - -Put description of model here as well as summary of what will/won't be in this document. Mention that there will be a separate user guide. Also, this doc isn't done until the list of authors is updated. Folks who joined after day 1 aren't included. - -%INCLUDE SECTIONS FROM EACH PROCESS -%graphicspath forces latex to look for figures in each of the listed directories -\graphicspath{{control/}{physics/psl/}{dynamics/homme/}{physics/shoc/}{physics/p3/}{physics/rrtmgp/}} -\input{control/driver_doc.tex} -\input{dynamics/homme/NHxx_doc.tex} -\input{physics/shoc/shoc_doc.tex} -\input{physics/p3/p3_doc.tex} -\input{physics/rrtmgp/rrtmgp_doc.tex} -\input{physics/psl/psl_doc.tex} - -%================================ -\section{Bibliography} -%================================ -\bibliography{biblio.bib} - -\end{document} diff --git a/components/eamxx/docs/amermeteorsoc.bst b/components/eamxx/docs/old/amermeteorsoc.bst similarity index 100% rename from components/eamxx/docs/amermeteorsoc.bst rename to components/eamxx/docs/old/amermeteorsoc.bst diff --git a/components/eamxx/docs/biblio.bib b/components/eamxx/docs/old/biblio.bib similarity index 100% rename from components/eamxx/docs/biblio.bib rename to components/eamxx/docs/old/biblio.bib diff --git a/components/eamxx/docs/control/driver_doc.tex b/components/eamxx/docs/old/control/driver_doc.tex similarity index 100% rename from components/eamxx/docs/control/driver_doc.tex rename to components/eamxx/docs/old/control/driver_doc.tex diff --git a/components/eamxx/docs/physics/psl/main.tex b/components/eamxx/docs/old/physics/psl/main.tex similarity index 100% rename from components/eamxx/docs/physics/psl/main.tex rename to components/eamxx/docs/old/physics/psl/main.tex diff --git a/components/eamxx/docs/physics/psl/psl_doc.tex b/components/eamxx/docs/old/physics/psl/psl_doc.tex similarity index 100% rename from components/eamxx/docs/physics/psl/psl_doc.tex rename to components/eamxx/docs/old/physics/psl/psl_doc.tex diff --git a/components/eamxx/docs/physics/shoc/main.tex b/components/eamxx/docs/old/physics/shoc/main.tex similarity index 100% rename from components/eamxx/docs/physics/shoc/main.tex rename to components/eamxx/docs/old/physics/shoc/main.tex diff --git a/components/eamxx/docs/physics/shoc/shoc_doc.tex b/components/eamxx/docs/old/physics/shoc/shoc_doc.tex similarity index 100% rename from components/eamxx/docs/physics/shoc/shoc_doc.tex rename to components/eamxx/docs/old/physics/shoc/shoc_doc.tex diff --git a/components/eamxx/docs/physics/p3/main.tex b/components/eamxx/docs/physics/p3/main.tex deleted file mode 100644 index 3c09baa03f22..000000000000 --- a/components/eamxx/docs/physics/p3/main.tex +++ /dev/null @@ -1,42 +0,0 @@ -\documentclass[12pt]{article} -\usepackage{authblk} -\bibliographystyle{../../amermeteorsoc} - -\title{Design Document for the Microphysics Scheme Used by SCREAM} - -\author[1]{Peter Caldwell} -\author[2]{Andy Salinger} -\author[2]{Luca Bertagna} -\author[1]{Hassan Beydoun} -\author[1]{Peter Bogenschutz} -\author[2]{Andrew Bradley} -\author[1]{Aaron Donahue} -\author[2]{Jim Foucar} -\author[1]{Chris Golaz} -\author[2]{Oksana Guba} -\author[2]{Ben Hillman} -\author[3]{Noel Keen} -\author[4]{Wuyin Lin} -\author[5]{Kyle Pressel} -\author[5]{Balwinder Singh} -\author[2]{Andrew Steyer} -\author[2]{Mark Taylor} -\author[1]{Chris Terai} -\author[6]{Paul Ullrich} -\date{\today} - -\affil[1]{Lawrence Livermore National Lab, Livermore CA} -\affil[2]{Sandia National Laboratories, Albuquerque, NM} -\affil[3]{Lawrence Berkeley National Laboratory, Berkeley, CA} -\affil[4]{Brookhaven National Laboratory, Upton, NY} -\affil[5]{Pacific Northwest National Laboratory, Richland, WA} -\affil[6]{University of California, Davis, Davis, CA} - -\begin{document} -\maketitle{} - -NOTE: author list for just this section should be culled to just the people working on this section. - -\input{p3_doc.tex} - -\end{document} diff --git a/components/eamxx/docs/physics/p3/p3_doc.tex b/components/eamxx/docs/physics/p3/p3_doc.tex deleted file mode 100644 index fb6b026fde09..000000000000 --- a/components/eamxx/docs/physics/p3/p3_doc.tex +++ /dev/null @@ -1,52 +0,0 @@ -\section{Predicted Particle Properties (P3)} - -Describe scheme in general (copy/paste Hassan's existing doc) - -%================================ -\subsection{Autoconversion} -%================================ - -Say what autoconversion does - -\subsubsection{Theory} - -Put explanation of continuous equations here. - -\subsubsection{Numerical Methods} - -Describe the numerical methods used to discretize the equations here. - -\subsubsection{Computational Implementation} - -Describe the strategies used (if any) to ensure good computational performance. - -\subsubsection{Verification} - -Describe testing strategy - -%================================ -\subsection{Accretion} -%================================ - -Say what accretion does - -\subsubsection{Theory} - -Put explanation of continuous equations here. - -\subsubsection{Numerical Methods} - -Describe the numerical methods used to discretize the equations here. - -\subsubsection{Computational Implementation} - -Describe the strategies used (if any) to ensure good computational performance. - -\subsubsection{Verification} - -Describe testing strategy - -%================================ -%... and so on... -%================================ - diff --git a/components/eamxx/docs/physics/rrtmgp/main.tex b/components/eamxx/docs/physics/rrtmgp/main.tex deleted file mode 100644 index b904d29ad47a..000000000000 --- a/components/eamxx/docs/physics/rrtmgp/main.tex +++ /dev/null @@ -1,41 +0,0 @@ -\documentclass[12pt]{article} -\usepackage{authblk} - -\title{Design Document for the RRTMGP Radiation Interface Used by SCREAM} - -\author[1]{Peter Caldwell} -\author[2]{Andy Salinger} -\author[2]{Luca Bertagna} -\author[1]{Hassan Beydoun} -\author[1]{Peter Bogenschutz} -\author[2]{Andrew Bradley} -\author[1]{Aaron Donahue} -\author[2]{Jim Foucar} -\author[1]{Chris Golaz} -\author[2]{Oksana Guba} -\author[2]{Ben Hillman} -\author[3]{Noel Keen} -\author[4]{Wuyin Lin} -\author[5]{Kyle Pressel} -\author[5]{Balwinder Singh} -\author[2]{Andrew Steyer} -\author[2]{Mark Taylor} -\author[1]{Chris Terai} -\author[6]{Paul Ullrich} -\date{\today} - -\affil[1]{Lawrence Livermore National Lab, Livermore CA} -\affil[2]{Sandia National Laboratories, Albuquerque, NM} -\affil[3]{Lawrence Berkeley National Laboratory, Berkeley, CA} -\affil[4]{Brookhaven National Laboratory, Upton, NY} -\affil[5]{Pacific Northwest National Laboratory, Richland, WA} -\affil[6]{University of California, Davis, Davis, CA} - -\begin{document} -\maketitle{} - -NOTE: author list for just this section should be culled to just the people working on this section. - -\input{rrtmgp_doc.tex} - -\end{document} diff --git a/components/eamxx/docs/physics/rrtmgp/rrtmgp_doc.tex b/components/eamxx/docs/physics/rrtmgp/rrtmgp_doc.tex deleted file mode 100644 index a9e24b784a1e..000000000000 --- a/components/eamxx/docs/physics/rrtmgp/rrtmgp_doc.tex +++ /dev/null @@ -1,18 +0,0 @@ -\section{Rapid Radiative Transfer for Global models in Parallel (RRTMGP)} - -\subsection{Theory} - -Put explanation of continuous equations here. - -\subsection{Numerical Methods} - -Describe the numerical methods used to discretize the equations here. - -\subsection{Computational Implementation} - -Describe the strategies used (if any) to ensure good computational performance. - -\subsection{Verification} - -Describe testing strategy - diff --git a/components/eamxx/docs/source-tree.md b/components/eamxx/docs/source-tree.md deleted file mode 100644 index f3ae49e2d23b..000000000000 --- a/components/eamxx/docs/source-tree.md +++ /dev/null @@ -1,58 +0,0 @@ -# SCREAM's Source Tree - -All SCREAM-specific code can be found in `components/eamxx` within the -[SCREAM repo](https://github.com/E3SM-Project/scream). Here's how things are -organized: - -+ `cime_config`: Tools and XML files for integrating SCREAM with E3SM via the - CIME framework. -+ `cmake`: CMake functions and macros used by the configuration/build system. -+ `data`: Data files used by our tests. -+ `docs`: Documentation for the SCREAM project, including design documents, - instructions for building and testing SCREAM, and this document. -+ `extern`: Source for certain lightweight third-party libraries, embedded - directly into the repo (and not imported as submodules). -+ `scripts`: Miscellaneous scripts that implement workflows for running tests - and analyzing performance. -+ `src`: All C++ source code (and any bridges to Fortran) for SCREAM are stored - here. We describe the contents of this directory in greater detail below. -+ `tests`: Implements standalone, end-to-end tests for various SCREAM - components (RRTMG, HOMME, P3, SHOC, etc). - -In addition, you'll notice the following files in `components/eamxx`: - -+ `CMakeLists.txt`: The CMake file that defines SCREAM's configuration/build - system. -+ `CTestConfig.cmake`: This CTest file contains parameters that determine how - our test results are reported to the [E3SM CDash Site](http://my.cdash.org/submit.php?project=E3SM). -+ `README.md`: SCREAM's top-level README file, which describes the project and - its purpose. - -## The `src` Directory - -Herein lіes the source code for SCREAM. Broadly, here's where things are: - -+ `control`: Contains the atmosphere driver and basic tests for it. -+ `dynamics`: Here's where HOMME lives within SCREAM, along with code for - interfacing with it using SCREAM's data structures. -+ `interface`: Glue code for embedding SCREAM within E3SM as an atmosphere - component. -+ `physics`: Source code for physics-related atmospheric processes, including - + `p3`: The C++/Kokkos implementation of P3 microphysics within SCREAM. - + `shoc`: The C++/Kokkos implementation of SHOC macrophysics within SCREAM. - + `rrtmgp`: A stub for the radiation processes as represented in SCREAM. - + `common`: Utilities and data structures common to these processes. -+ `share`: Utilities used by various components within SCREAM. A lot of things - here will likely end up in `ekat`. - -Each of these directories contains a `CMakeLists.txt` file for defining how -things are build, and a `tests/` subdirectory that houses relevant -unit and verification tests. - -You'll also see some other files in the `src/` directory itself: - -+ `scream_config.f.in`: A template for generating a Fortran include file with - SCREAM configuration information. -+ `scream_config.h.in`: A template for generating a C++ header file with - SCREAM configuration information. - diff --git a/components/eamxx/docs/technical/aerocom_cldtop.md b/components/eamxx/docs/technical/aerocom_cldtop.md new file mode 100644 index 000000000000..18fea456cf6c --- /dev/null +++ b/components/eamxx/docs/technical/aerocom_cldtop.md @@ -0,0 +1,27 @@ +# The AeroCOM algorithm + +The goal of the AeroCOM algorithm is to calculate properties at cloud top based on the AeroCOM recommendation. There are two main parts of the algorithm: probabilistically determining "cloud top" and then "calculating properties" at said cloud top. + +We treat model columns independently, so we loop over all columns in parallel. We then loop over all layers in serial (due to needing an accumulative product), starting at 2 (second highest) layer because the highest is assumed to have no clouds. Let's take a photonic approach from above the model top. Let's say that $p_{k}$ is the probability of a photon passing through the layer $k$. We follow the maximum-random overlap assumption. In all cases, we assume the cloudiness (or cloudy fraction) is completely opaque. + +We assume the highest layer has no clouds, thus the $p_{k} = 1$ for the highest layer. Note that $p_{k}$ is initialized as 1 for all layers. We also clip the cloudy fraction $C_{i,k}$ to ensure that $C_{i,k} \in [0+\epsilon, 1-\epsilon]$, where $\epsilon = 0.001$. Starting at the second highest layer, $k+1$, we check if some "cloudy" conditions are met. These conditions are now arbitrarily defined by a cloudiness threshold of $\epsilon$ (i.e., $C_{i,k}>\epsilon$) and a non-zero threshold on the total (both liquid and ice) droplet number concentration (i.e., $cQ_{i,k} + iQ_{i,k} > 0$). If the conditions are met, we estimate the cloud-top cloud fraction using an accumulative product following the maximum-random overlap assumption. + +$$c_{i} = 1 - \prod_{k=2}^{K} p_{k} = 1 - \prod_{k=2}^{K} \frac{1 - \max(C_{i,k}, C_{i,k-1})}{1-C_{i,k-1}}$$ + +In order to estimate cloud-top properties, we weight by the probability of "remaining cloudiness" or $p_{k-1} - p_{k}$. + +| Type | Equation | +| --- | --------- | +| cloud property | $x_{i} = \sum_{k=2}^{K} X_{i,k} \Phi_{i,k} (p_{k-1} - p_{k})$ | +| cloud content | $x_{i} = \sum_{k=2}^{K} \Phi_{i,k} (p_{k-1} - p_{k})$ | +| other property | $x_{i} = \sum_{k=2}^{K} X_{i,k} (p_{k-1} - p_{k})$ | + +In the above, $\Phi_{i,k}$ is the thermodynamic phase defined by the cloud droplet number concentration ratios. + +$$i\Phi_{i,k} = \frac{iQ_{i,k}}{iQ_{i,k} + cQ_{i,k}}$$ + +$$c\Phi_{i,k} = \frac{cQ_{i,k}}{iQ_{i,k} + cQ_{i,k}}$$ + +The thermodynamic phase is used only for cloud properties (e.g., cloud-top cloud droplet number concentration) or cloud content (e.g., cloud liquid content). Further, $X_{i,k}$ is the three-dimensional cloud property of interest which is needed if we are converting a property from three-dimensional ($X$) to its two-dimensional counterpart ($x$). "Other" properties here include temperature and pressure which are not dependent on the thermodynamic phase. + +A helpful references: Räisänen, P., Barker, H. W., Khairoutdinov, M. F., Li, J., & Randall, D. A. (2004). Stochastic generation of subgrid‐scale cloudy columns for large‐scale models. Quarterly Journal of the Royal Meteorological Society: A journal of the atmospheric sciences, applied meteorology and physical oceanography, 130(601), 2047-2067. diff --git a/components/eamxx/docs/technical/clean_clear_sky.md b/components/eamxx/docs/technical/clean_clear_sky.md new file mode 100644 index 000000000000..82380f903156 --- /dev/null +++ b/components/eamxx/docs/technical/clean_clear_sky.md @@ -0,0 +1,10 @@ +# Clean- and clean-clear-sky diagnostics + +In order to decompose the aerosol effective radiative forcing, additional diagnostic radiation calls are needed. +These extra diagnostics are optionally added to the main radiation call. The extra diagnostics are: + +- Clean-clear-sky fluxes: the fluxes that would be present if there were neither aerosols nor clouds, and are calculated by adding an additional radiation call at the very beginning of the logic before the optics class is endowed with aerosol and cloud properties. +- Clean-sky fluxes: the fluxes that would be present if there were no aerosols, and are calculated by adding an additional radiation call after substantiating an additional optics class, but not endowing it with aerosol properties. + +It was necessary to add an additional optics class because the original optics class is endowed with aerosols before clouds (in order to calculate the clear-sky fluxes). +The extra calls are controlled by runtime flags `extra_clnclrsky_diag` and `extra_clnsky_diag` (they take either `true` or `false` as their values). diff --git a/components/eamxx/docs/technical/index.md b/components/eamxx/docs/technical/index.md new file mode 100644 index 000000000000..8b9a45be4fdf --- /dev/null +++ b/components/eamxx/docs/technical/index.md @@ -0,0 +1,3 @@ +# SCREAM Technical Guide + +SCREAM contributors and maintainers will add detailed technical information about SCREAM here. diff --git a/components/eamxx/docs/user/coarse_nudging.md b/components/eamxx/docs/user/coarse_nudging.md new file mode 100644 index 000000000000..c52ce9a0eb24 --- /dev/null +++ b/components/eamxx/docs/user/coarse_nudging.md @@ -0,0 +1,25 @@ +# Nudging from coarse data + +Because EAMxx is designed to support ultra-high resolutions (in fact, that was the initial reason for its inception), it is not feasible to produce nudging data at the same resolution. +Instead, in EAMxx, it is possible to nudge from coarse data. +This is done by remapping the coarse data provided by the user to the runtime physics grid of EAMxx. +In order to enable nudging from coarse data, the user must provide nudging data at the coarse resolution desired and an appropriate ncremap-compatible mapping file. + +## Example setup + +A user can produce coarse nudging data from running EAMxx or EAM at a ne30pg2 or any other applicable resolution. +Additionally, several users in the E3SM projects have produced nudging data at the ne30pg2 resolution from the MERRA2 and ERA5 datasets. +A limitation for now is that the nudging data must be provided explicitly, either as one file or as a list of files. +This can be problematic for long list of files, but we are working on a solution to this problem. + +Let's say that the nudging data is provided as one file in the following path: `/path/to/nudging_data_ne4pg2_L72.nc`. +Then, a mapping file is provided as `/another/path/to/mapping_file_ne4pg2_to_ne120pg2.nc`. +Then if the physics grid is ne120pg2, the user must enable the nudging process, specify the nudging files, and provide the specifies the nudging data and a remap file. +In other words, the following options are needed: + +```shell +./atmchange atm_procs_list=(sc_import,nudging,homme,physics,sc_export) +./atmchange nudging_fields=U,V +./atmchange nudging_filename=/path/to/nudging_data_ne4pg2_L72.nc +./atmchange nudging_refine_remap_mapfile=/another/path/to/mapping_file_ne4pg2_to_ne120pg2.nc +``` diff --git a/components/eamxx/docs/user/index.md b/components/eamxx/docs/user/index.md new file mode 100644 index 000000000000..ba53083fc75c --- /dev/null +++ b/components/eamxx/docs/user/index.md @@ -0,0 +1,3 @@ +# SCREAM User Guide + +For the time being, see our [public confluence EAMxx user guide](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3858890786/EAMxx+User+s+Guide) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md new file mode 100644 index 000000000000..38486c77a21e --- /dev/null +++ b/components/eamxx/docs/user/model_input.md @@ -0,0 +1,8 @@ +Model input +===================================== + +TODO: explain how defaults XML, atmchange/atmquery, buildml, and input.yaml work. + +[Here](../common/eamxx_params.md) is a list of the currently configurable runtime parameters for EAMxx. + + diff --git a/components/eamxx/docs/user/model_output.md b/components/eamxx/docs/user/model_output.md new file mode 100644 index 000000000000..1ef0c8b26761 --- /dev/null +++ b/components/eamxx/docs/user/model_output.md @@ -0,0 +1,153 @@ +# Model output + +EAMxx allows the user to configure the desired model output via [YAML](https://yaml.org/) files, +with each YAML file associated to a different output file. + +## Basic output YAML file syntax + +The following is an example of a simple output request. + +```yaml +%YAML 1.1 +--- +filename_prefix: my_output +Averaging Type: Average +Max Snapshots Per File: 10 +Fields: + Physics: + Field Names: + - T_mid + - qv + Dynamics: + Field Names: + - dp3d_dyn + - omega_dyn +output_control: + Frequency: 6 + frequency_units: nhours +``` + +Notice that lists can be equivalently specified in YAML as `Field Names: [f1, f2, f3]`. +The user can specify fields to be outputted from any of the grids used in the simulation. +In the example above, we requested fields from both the Physics and Dynamics grid. +The other parameters are + +- `Averaging Type`: how the fields are integrated in time before being saved. Valid + options are + + - Instant: no integration, each time frame saved corresponds to instantaneous values + of the fields + - Average/Max/Min: the fields undergo the corresponding operation over the time + interval specified in the `output_control` section. In the case above, each snapshot + saved to file corresponds to an average of the output fields over 6h windows. + +- `filename_prefix`: the prefix of the output file, which will be created in the run + directory. The full filename will be `$prefix.$avgtype.$frequnits_x$freq.$timestamp.nc`, + where $timestamp corresponds to the first snapshot saved in the file for Instant output, + or the beginning of the first averaging window for the other averaging types +- `Max Snapshots Per File`: specifies how many time snapshots can be put in a file. Once + this number is reached, EAMxx will close the file and open a new one. +- `Frequency`: how many units of time are between two consecutive writes to file. For + Instant output the fields are "sampled" at this frequency, while for other averaging + types the fields are "integrated" in time over this window +- `frequency_units`: units of the output frequency. Valid options are `nsteps` (the + number of atmosphere time steps), `nsecs`, `nmins`, `nhours`, `ndays`, `nmonths`, + `nyears`. + +## Diagnostic output + +In addition to the fields computed by EAMxx as part of the timestep, the user can +request to output derived quantities, which will be computed on the fly by the +I/O interface of EAMxx. There are two types of diagnostic outputs: + +- quantities computed as a function of EAMxx fields. These are simply physical quantities + that EAMxx does not keep in persistent storage. As of August 2023, the available + derived quantities are (case sensitive): + + - `PotentialTemperature` + - `AtmosphereDensity` + - `Exner` + - `VirtualTemperature` + - `z_int` + - `z_mid` + - `geopotential_int` + - `geopotential_mid` + - `dz` + - `DryStaticEnergy` + - `SeaLevelPressure` + - `LiqWaterPath` + - `IceWaterPath` + - `VapWaterPath` + - `RainWaterPath` + - `RimeWaterPath` + - `ShortwaveCloudForcing` + - `LongwaveCloudForcing` + - `RelativeHumidity` + - `ZonalVapFlux` + - `MeridionalVapFlux` + - `precip_liq_surf_mass_flux` + - `precip_ice_surf_mass_flux` + - `precip_total_surf_mass_flux` + - `surface_upward_latent_heat_flux` + +- lower-dimensional slices of a field. These are hyperslices of an existing field or of + another diagnostic output. As of August 2023, given a field X, the available options + are: + + - `X_at_lev_N`: slice the field `X` at the N-th vertical level index. Recall that + in EAMxx N=0 corresponds to the model top. + - `X_at_model_bot`, `X_at_model_top`: special case for top and bottom of the model. + - `X_at_Ymb`, `X_at_YPa`, `X_at_YhPa`: interpolates the field `X` at a vertical position + specified by the give pressure `Y`. Available units are `mb` (millibar), `Pa`, and `hPa`. + - `X_at_Ym`: interpolates the field `X` at a vertical height of `Y` meters. + +## Remapped output + +The following options can be used to to save fields on a different grid from the one +they are computed on. + +- `horiz_remap_file`: a path to a map file (as produced by `ncremap`) between the grid + where the fields are defined and a coarser grid. EAMxx will use this to remap fields + on the fly, allowing to reduce the size of the output file. Note: with this feature, + the user can only specify fields from a single grid. +- `vertical_remap_file`: similar to the previous option, this map file is used to + refine/coarsen fields in the vertical direction. +- `IOGrid`: this parameter can be specified inside one of the grids sections, and will + denote the grid (which must exist in the simulation) where the fields must be remapped + before being saved to file. This feature is really only used to save fields on the + dynamics grid without saving twice the DOFs at the interface of two spectral elements. + In fact, native output from the Dynamics grid would produce `6*num_elems*ngp*ngp`, + where `ngp` is the number of Gauss points along each axis in the 2d spectral element. + Note: this feature cannot be used along with the horizontal/vertical remapper. + +## Add output stream to a CIME case + +In order to tell EAMxx that a new output stream is needed, one must add the name of +the yaml file to be used to the list of yaml files that EAMxx will process. From the +case folder, after `case.setup` has run, one can do + +```shell +./atmchange output_yaml_files=/path/to/my/yaml/file +``` + +to specify a single yaml file, or + +```shell +./atmchange output_yaml_files+=/path/to/my/yaml/file +``` + +to append to the list of yaml files. + +### Important notes + +- The user should not specify a path to a file in `$RUNDIR/data`. EAMxx will +put a copy of the specified yaml files in that directory, pruning any existing copy +of that file. This happens every time that `buildnml` runs; in particular, it happens +during `case.submit`. +- As a consequence of the above, the user should not modify the generated yaml files + that are in `$RUNDIR/data`, since any modification will be lost on the next run + of `buildnml`. To modify output parmeters, the user should modify the yaml file + that was specified with the `atmchange` command. +- EAMxx will parse the yaml file and expand any string of the form $VAR, by looking + for the value of the variable VAR in the CIME case. If VAR is not a valid CIME + variable, an error will be raised. diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml new file mode 100644 index 000000000000..dde0970a4ed2 --- /dev/null +++ b/components/eamxx/mkdocs.yml @@ -0,0 +1,67 @@ +site_name: EAMxx + +nav: + - 'Home': 'index.md' + - 'User Guide': + - 'Overview': 'user/index.md' + - 'Installation': 'common/installation.md' + - 'Model output': 'user/model_output.md' + - 'Model input': 'user/model_input.md' + - 'Runtime parameters': 'common/eamxx_params.md' + - 'Coarse nudging': 'user/coarse_nudging.md' + - 'Developer Guide': + - 'Overview': 'developer/index.md' + - 'Installation': 'common/installation.md' + - 'Style Guide': 'developer/style_guide.md' + - 'Kokkos and EKAT': 'developer/kokkos_ekat.md' + - 'Source Tree': 'developer/source_tree.md' + - 'Important Data Structures': + - 'Fields': 'developer/field.md' + - 'Grids and Remappers': 'developer/grid.md' + - 'Atmosphere Processes': 'developer/processes.md' + - 'Managers': 'developer/managers.md' + - 'I/O': 'developer/io.md' + - 'Testing': + - 'Standalone': 'developer/standalone_testing.md' + - 'Full model': 'developer/cime_testing.md' + - 'CI and Nightly Testing': 'developer/ci_nightly.md' + - 'Technical Guide': + - 'AeroCOM cloud top': 'technical/aerocom_cldtop.md' + - 'Extra radiation calls': 'technical/clean_clear_sky.md' + +edit_uri: "" + +theme: + name: material + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + toggle: + icon: material/weather-sunny + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/weather-night + name: Switch to light mode + features: + - navigation.indices + - navigation.instant + - navigation.sections + - navigation.top +# - navigation.tabs + +markdown_extensions: + - pymdownx.highlight + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.arithmatex: + generic: true + +extra_javascript: + # - javascript/mathjax.js + - https://polyfill.io/v3/polyfill.min.js?features=es6 + - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js + +repo_url: https://github.com/E3SM-Project/scream diff --git a/components/eamxx/scripts/CMakeLists.txt b/components/eamxx/scripts/CMakeLists.txt index dbf93bebed6d..3c7ab6eba5ff 100644 --- a/components/eamxx/scripts/CMakeLists.txt +++ b/components/eamxx/scripts/CMakeLists.txt @@ -9,4 +9,4 @@ add_executable(query-cf-database query-cf-database.cpp) target_compile_definitions(query-cf-database PUBLIC CF_STANDARD_NAME_FILE=${CF_STANDARD_NAME_FILE} CF_SCREAM_NAME_FILE=${CF_SCREAM_NAME_FILE}) -target_link_libraries(query-cf-database ekat) +target_link_libraries(query-cf-database ekat yaml-cpp) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index c656119adb2f..2a763bc1990d 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -2,178 +2,285 @@ Retrieve nodes from EAMxx XML config file. """ -import sys, os +import sys, os, re # Used for doctests import xml.etree.ElementTree as ET # pylint: disable=unused-import # Add path to cime_config folder sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "cime_config")) -from eamxx_buildnml_impl import check_value, is_array_type -from utils import expect +from eamxx_buildnml_impl import check_value, is_array_type, get_child, find_node +from eamxx_buildnml_impl import gen_atm_proc_group +from utils import expect, run_cmd_no_fail + +ATMCHANGE_SEP = "-ATMCHANGE_SEP-" +ATMCHANGE_ALL = "__ALL__" +ATMCHANGE_BUFF_XML_NAME = "SCREAM_ATMCHANGE_BUFFER" + +############################################################################### +def apply_atm_procs_list_changes_from_buffer(case, xml): +############################################################################### + atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) + if atmchg_buffer: + atmchgs, atmchgs_all = unbuffer_changes(case) + + expect (len(atmchgs)==len(atmchgs_all),"Failed to unbuffer changes from SCREAM_ATMCHANGE_BUFFER") + for chg, to_all in zip(atmchgs,atmchgs_all): + if "atm_procs_list" in chg: + expect (not to_all, "Makes no sense to change 'atm_procs_list' for all groups") + atm_config_chg_impl(xml, chg, all_matches=False) ############################################################################### -class AmbiguousName (Exception): - pass +def apply_non_atm_procs_list_changes_from_buffer(case, xml): ############################################################################### + atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) + if atmchg_buffer: + atmchgs, atmchgs_all = unbuffer_changes(case) + + expect (len(atmchgs)==len(atmchgs_all),"Failed to unbuffer changes from SCREAM_ATMCHANGE_BUFFER") + for chg, to_all in zip(atmchgs,atmchgs_all): + if "atm_procs_list" not in chg: + atm_config_chg_impl(xml, chg, all_matches=to_all) ############################################################################### -def num_nodes_with_name (root,name,recurse=True): +def buffer_changes(changes, all_matches=False): ############################################################################### """ - Count nodes with certain name in an XML tree + Take a list of raw changes and buffer them in the XML case settings. Raw changes + are what goes to atm_config_chg_impl. + """ + # Commas confuse xmlchange and so need to be escaped. + if all_matches: + changes_temp = [c + ATMCHANGE_ALL for c in changes] + changes_str = ATMCHANGE_SEP.join(changes_temp).replace(",",r"\,") + else: + # changes_str += f"{ATMCHANGE_SEP}--all" + changes_str = ATMCHANGE_SEP.join(changes).replace(",",r"\,") - >>> xml = ''' - ... - ... - ... - ... - ... - ... - ... ''' - >>> import xml.etree.ElementTree as ET - >>> tree = ET.fromstring(xml) - >>> num_nodes_with_name(tree,'a',recurse=False) - 1 - >>> num_nodes_with_name(tree,'a',recurse=True) - 2 + run_cmd_no_fail(f"./xmlchange --append {ATMCHANGE_BUFF_XML_NAME}='{changes_str}{ATMCHANGE_SEP}'") + +############################################################################### +def unbuffer_changes(case): +############################################################################### + """ + From a case, get a list of raw changes. Returns (changes, all_matches_flag) """ + atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) + atmchgs = [] + atmchgs_all = [] + for item in atmchg_buffer.split(ATMCHANGE_SEP): + if item.strip(): + atmchgs_all.append(ATMCHANGE_ALL in item) + atmchgs.append(item.replace(ATMCHANGE_ALL,"").replace(r"\,", ",").strip()) - count = 0 + return atmchgs, atmchgs_all - for elem in root: - if elem.tag==name: - count += 1 - if recurse: - count += num_nodes_with_name(elem,name) - return count +############################################################################### +def reset_buffer(): +############################################################################### + run_cmd_no_fail(f"./xmlchange {ATMCHANGE_BUFF_XML_NAME}=''") ############################################################################### -def find_node (root,name,recurse=True): +def get_xml_nodes(xml_root, name): ############################################################################### """ + Find all elements matching a name where name uses '::' syntax + >>> xml = ''' ... + ... one ... - ... 2 + ... two + ... 2 ... ... ... ''' >>> import xml.etree.ElementTree as ET >>> tree = ET.fromstring(xml) - >>> a,parents = find_node(tree,'a') - >>> print(f"{','.join(p.tag for p in parents)}") - root,sub - >>> print(a.text) - 2 - >>> print(len(parents)) - 2 - >>> print(f"{','.join(p.tag for p in parents)}") - root,sub - >>> a,parents = find_node(tree,'a',recurse=False) - >>> print(a) - None + >>> ################ INVALID SYNTAX ####################### + >>> get_xml_nodes(tree,'sub::::prop1') + Traceback (most recent call last): + SystemExit: ERROR: Invalid xml node name format, 'sub::::prop1' contains :::: + >>> ################ VALID USAGE ####################### + >>> get_xml_nodes(tree,'invalid::prop1') + [] + >>> [item.text for item in get_xml_nodes(tree,'prop1')] + ['one', 'two'] + >>> [item.text for item in get_xml_nodes(tree,'::prop1')] + ['one'] + >>> [item.text for item in get_xml_nodes(tree,'prop2')] + ['2'] + >>> item = get_xml_nodes(tree,'prop2')[0] + >>> parent_map = create_parent_map(tree) + >>> [p.tag for p in get_parents(item, parent_map)] + ['root', 'sub'] """ + expect("::::" not in name, f"Invalid xml node name format, '{name}' contains ::::") + + if name.startswith("::"): + prefix = "./" # search immediate children only + name = name[2:] + else: + prefix = ".//" # search entire tree - for elem in root: - if elem.tag==name: - return elem, [root] - if len(elem)>0 and recurse: - found, parents = find_node(elem,name,recurse=True) - if found is not None: - return found, [root] + parents + try: + xpath_str = prefix + name.replace("::", "/") + result = xml_root.findall(xpath_str) + except SyntaxError as e: + expect(False, f"Invalid syntax '{name}' -> {e}") - return None, [] + return result ############################################################################### -def get_xml_node(xml_root,name): +def modify_ap_list(xml_root, group, ap_list_str, append_this): ############################################################################### """ + Modify the atm_procs_list entry of this XML node (which is an atm proc group). + This routine can only be used to add an atm proc group OR to remove some + atm procs. >>> xml = ''' ... - ... one - ... - ... two - ... 2 - ... + ... + ... + ... + ... + ... + ... 1 + ... + ... + ... 2 + ... + ... ... ... ''' + >>> from eamxx_buildnml_impl import has_child >>> import xml.etree.ElementTree as ET >>> tree = ET.fromstring(xml) - >>> ################ INVALID SYNTAX ####################### - SystemExit: ERROR: Invalid change format. Expected A[::B[...]=value, got' prop1->2' - >>> get_xml_node(tree,'sub::::prop1') - Traceback (most recent call last): - SystemExit: ERROR: Invalid xml node name format. Expected A[::B[...], got' sub::::prop1' - Did you put two '::' in a row? - >>> ################ INVALID NAMESPACE ####################### - >>> get_xml_node(tree,'invalid::prop1') - Traceback (most recent call last): - SystemExit: ERROR: Error! XML entry invalid not found in section root - >>> ################ AMBIGUOUS ENTRY ####################### - >>> get_xml_node(tree,'prop1') + >>> node = ET.Element("my_group") + >>> node.append(ET.Element("atm_procs_list")) + >>> get_child(node,"atm_procs_list").text = "" + >>> modify_ap_list(tree,node,"p1,p2",False) + True + >>> get_child(node,"atm_procs_list").text + 'p1,p2' + >>> modify_ap_list(tree,node,"p1",True) + True + >>> get_child(node,"atm_procs_list").text + 'p1,p2,p1' + >>> modify_ap_list(tree,node,"p1,p3",False) Traceback (most recent call last): - atm_manip.AmbiguousName: ERROR: Error! Multiple XML entries with name prop1 found in section root - >>> ################ VALID USAGE ####################### - >>> n,p = get_xml_node(tree,'::prop1') - >>> print(n.text) - one - >>> print(len(p)) - 1 - >>> print(p[0].tag) - root - >>> n,p = get_xml_node(tree,'prop2') - >>> print(n.text) - 2 - >>> m,p = get_xml_node(tree,'prop2') - >>> print([k for k in n.attrib.keys()]) - ['type', 'valid_values'] - >>> print(len(p)) - 2 - >>> print(f"{','.join(e.tag for e in p)}") - root,sub + ValueError: ERROR: Unrecognized atm proc name 'p3'. To declare a new group, prepend and append '_' to the name. + >>> modify_ap_list(tree,node,"p1,_my_group_",False) + True + >>> get_child(node,"atm_procs_list").text + 'p1,_my_group_' + >>> defaults = get_child(tree,'atmosphere_processes_defaults') + >>> has_child(defaults,'_my_group_') + True """ + curr_apl = get_child(group,"atm_procs_list") + if curr_apl.text==ap_list_str: + return False + + ap_list = ap_list_str.split(",") + expect (len(ap_list)==len(set(ap_list)), + "Input list of atm procs contains repetitions") + + # If we're here b/c of a manual call of atmchange from command line, this will be None, + # since we don't have this node in the genereated XML file. But in that case, we don't + # have to actually add the new nodes, we can simply just modify the atm_procs_list entry + # If, however, we're calling this from buildnml, then what we are passed in is the XML + # tree from namelists_defaults_scream.xml, so this section *will* be present. And we + # need to add the new atm procs group as children, so that buildnml knows how to build + # them + ap_defaults = find_node(xml_root,"atmosphere_processes_defaults") + if ap_defaults is not None: + + # Figure out which aps in the list are new groups and which ones already + # exist in the defaults + add_aps = [n for n in ap_list if n not in curr_apl.text.split(',')] + new_aps = [n for n in add_aps if find_node(ap_defaults,n) is None] + + for ap in new_aps: + expect (ap[0]=="_" and ap[-1]=="_" and len(ap)>2, exc_type=ValueError, + error_msg=f"Unrecognized atm proc name '{ap}'. To declare a new group, prepend and append '_' to the name.") + group = gen_atm_proc_group("", ap_defaults) + group.tag = ap + + ap_defaults.append(group) + + # Update the 'atm_procs_list' in this node + if append_this: + curr_apl.text = ','.join(curr_apl.text.split(",")+ap_list) + else: + curr_apl.text = ','.join(ap_list) + return True - selectors = name.split("::") +############################################################################### +def apply_change(xml_root, node, new_value, append_this): +############################################################################### + any_change = False - # Allow :: at the beginning (as in '::A::b'), but do not allow multiple :: operators - expect('' not in selectors[1:], - "Invalid xml node name format. Expected A[::B[...], got' {}'\n".format(name) + - " Did you put two '::' in a row?") + # User can change the list of atm procs in a group doing ./atmchange group_name=a,b,c + # If we detect that this node is an atm proc group, don't modify the text, but do something els + if node.tag=="atm_procs_list": + parent_map = create_parent_map(xml_root) + group = get_parents(node,parent_map)[-1] + return modify_ap_list (xml_root,group,new_value,append_this) - # Regardless of whether we have namespaces or not, the first selector must be unique through the whole XML tree - s = selectors[0] - if s == '': - # User started with :: - node = xml_root - parents = [] - else: - expect (num_nodes_with_name(xml_root,s,recurse=True)>0, - "Error! XML entry {} not found in section {}".format(s,xml_root.tag)) - expect (num_nodes_with_name(xml_root,s,recurse=True)==1, - "Error! Multiple XML entries with name {} found in section {}" - .format(s,xml_root.tag), AmbiguousName) + if append_this: - node, parents = find_node(xml_root,s,recurse=True) + expect ("type" in node.attrib.keys(), + f"Error! Missing type information for {node.tag}") + type_ = node.attrib["type"] + expect (is_array_type(type_) or type_=="string", + "Error! Can only append with array and string types.\n" + f" - name: {node.tag}\n" + f" - type: {type_}") + if is_array_type(type_): + node.text += ", " + new_value + else: + node.text += new_value - # If user specified selectors via namespace, recurse over them - for s in selectors[1:]: - expect (num_nodes_with_name(node,s,recurse=False)>0, - "Error! XML entry {} not found in section {}".format(s,node.tag)) - expect (num_nodes_with_name(node,s,recurse=False)==1, - "Error! Multiple XML entries with name {} found in section {}" - .format(s,node.tag)) + any_change = True - node, parents = find_node(node,s,recurse=False) + elif node.text != new_value: + check_value(node,new_value) + node.text = new_value + any_change = True - return node, parents + return any_change ############################################################################### -def atm_config_chg_impl(xml_root,changes): +def parse_change(change): ############################################################################### """ + >>> parse_change("a+=2") + ('a', '2', True) + >>> parse_change("a=hello") + ('a', 'hello', False) + """ + tokens = change.split('+=') + if len(tokens)==2: + append_this = True + else: + append_this = False + tokens = change.split('=') + + expect (len(tokens)==2, + f"Invalid change request '{change}'. Valid formats are:\n" + f" - A[::B[...]=value\n" + f" - A[::B[...]+=value (implies append for this change)") + node_name = tokens[0] + new_value = tokens[1] + + return node_name,new_value,append_this +############################################################################### +def atm_config_chg_impl(xml_root, change, all_matches=False): +############################################################################### + """ >>> xml = ''' ... ... 1,2,3 @@ -191,90 +298,141 @@ def atm_config_chg_impl(xml_root,changes): >>> import xml.etree.ElementTree as ET >>> tree = ET.fromstring(xml) >>> ################ INVALID SYNTAX ####################### - >>> atm_config_chg_impl(tree,['prop1->2']) + >>> atm_config_chg_impl(tree,'prop1->2') Traceback (most recent call last): SystemExit: ERROR: Invalid change request 'prop1->2'. Valid formats are: - A[::B[...]=value - A[::B[...]+=value (implies append for this change) >>> ################ INVALID TYPE ####################### - >>> atm_config_chg_impl(tree,['prop2=two']) + >>> atm_config_chg_impl(tree,'prop2=two') Traceback (most recent call last): - ValueError: Could not use 'two' as type 'integer' + ValueError: Could not refine 'two' as type 'integer' >>> ################ INVALID VALUE ####################### - >>> atm_config_chg_impl(tree,['prop2=3']) + >>> atm_config_chg_impl(tree,'prop2=3') Traceback (most recent call last): CIME.utils.CIMEError: ERROR: Invalid value '3' for element 'prop2'. Value not in the valid list ('[1, 2]') + >>> ################ AMBIGUOUS CHANGE ####################### + >>> atm_config_chg_impl(tree,'prop1=three') + Traceback (most recent call last): + SystemExit: ERROR: prop1 is ambiguous (use --all to change all matches), matches: + root::prop1 + root::sub::prop1 + >>> ################ VALID USAGE ####################### - >>> atm_config_chg_impl(tree,['::prop1=two']) + >>> atm_config_chg_impl(tree,'::prop1=two') True - >>> atm_config_chg_impl(tree,['::prop1=two']) + >>> atm_config_chg_impl(tree,'::prop1=two') False - >>> atm_config_chg_impl(tree,['sub::prop1=one']) + >>> atm_config_chg_impl(tree,'sub::prop1=one') + True + >>> atm_config_chg_impl(tree,'prop1=three', all_matches=True) True + >>> [item.text for item in get_xml_nodes(tree,'prop1')] + ['three', 'three'] >>> ################ TEST APPEND += ################# - >>> atm_config_chg_impl(tree,['a+=4']) + >>> atm_config_chg_impl(tree,'a+=4') True - >>> get_xml_node(tree,'a')[0].text + >>> get_xml_nodes(tree,'a')[0].text '1,2,3, 4' >>> ################ ERROR, append to non-array and non-string - >>> atm_config_chg_impl(tree,['c+=2']) + >>> atm_config_chg_impl(tree,'c+=2') Traceback (most recent call last): SystemExit: ERROR: Error! Can only append with array and string types. - name: c - type: int >>> ################ Append to string ################## - >>> atm_config_chg_impl(tree,['d+=two']) + >>> atm_config_chg_impl(tree,'d+=two') True - >>> get_xml_node(tree,'d')[0].text + >>> get_xml_nodes(tree,'d')[0].text 'onetwo' >>> ################ Append to array(string) ################## - >>> atm_config_chg_impl(tree,['e+=two']) + >>> atm_config_chg_impl(tree,'e+=two') True - >>> get_xml_node(tree,'e')[0].text + >>> get_xml_nodes(tree,'e')[0].text 'one, two' """ + node_name, new_value, append_this = parse_change(change) + matches = get_xml_nodes(xml_root, node_name) - any_change = False - for change in changes: + expect(len(matches) > 0, f"{node_name} did not match any items") - tokens = change.split('+=') - if len(tokens)==2: - append_this = True - else: - append_this = False - tokens = change.split('=') - - expect (len(tokens)==2, - f"Invalid change request '{change}'. Valid formats are:\n" - f" - A[::B[...]=value\n" - f" - A[::B[...]+=value (implies append for this change)") - node, __ = get_xml_node(xml_root,tokens[0]) - new_value = tokens[1] - - if append_this: - expect ("type" in node.attrib.keys(), - "Error! Missing type information for {}".format(tokens[0])) - type_ = node.attrib["type"] - expect (is_array_type(type_) or type_=="string", - "Error! Can only append with array and string types.\n" - f" - name: {tokens[0]}\n" - f" - type: {type_}") - if is_array_type(type_): - node.text += ", " + new_value - else: - node.text += new_value + if len(matches) > 1 and not all_matches: + parent_map = create_parent_map(xml_root) + error_str = "" + for node in matches: + parents = get_parents(node, parent_map) + name = "::".join(e.tag for e in parents) + "::" + node.tag + error_str += " " + name + "\n" - any_change = True + expect(False, f"{node_name} is ambiguous (use --all to change all matches), matches:\n{error_str}") - elif node.text != new_value: - check_value(node,new_value) - node.text = new_value - any_change = True + any_change = False + for node in matches: + any_change |= apply_change(xml_root, node, new_value, append_this) return any_change ############################################################################### -def print_var(xml_root,var,full,dtype,value,valid_values,print_style="invalid",indent=""): +def create_parent_map(root): +############################################################################### + return {c: p for p in root.iter() for c in p} + +############################################################################### +def get_parents(elem, parent_map): +############################################################################### + """ + Return all parents of an elem in descending order (first item in list will + be the furthest ancestor, last item will be direct parent) + """ + results = [] + if elem in parent_map: + parent = parent_map[elem] + results = get_parents(parent, parent_map) + [parent] + + return results + +############################################################################### +def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="invalid",indent=""): +############################################################################### + + expect (print_style in ["short","full"], + f"Invalid print_style '{print_style}' for print_var_impl. Use 'full' or 'short'.") + + if print_style=="short": + # Just the inner most name + name = node.tag + else: + parents = get_parents(node, parent_map) + name = "::".join(e.tag for e in parents) + "::" + node.tag + + if full: + expect ("type" in node.attrib.keys(), + f"Error! Missing type information for {name}") + print (f"{indent}{name}") + print (f"{indent} value: {node.text}") + print (f"{indent} type: {node.attrib['type']}") + if "valid_values" not in node.attrib.keys(): + valid = [] + else: + valid = node.attrib["valid_values"].split(",") + print (f"{indent} valid values: {valid}") + elif dtype: + expect ("type" in node.attrib.keys(), + f"Error! Missing type information for {name}") + print (f"{indent}{name}: {node.attrib['type']}") + elif value: + print (f"{indent}{node.text}") + elif valid_values: + if "valid_values" not in node.attrib.keys(): + valid = '' + else: + valid = node.attrib["valid_values"].split(",") + print (f"{indent}{name}: {valid}") + else: + print (f"{indent}{name}: {node.text}") + +############################################################################### +def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style="invalid",indent=""): ############################################################################### """ >>> xml = ''' @@ -288,29 +446,27 @@ def print_var(xml_root,var,full,dtype,value,valid_values,print_style="invalid",i ... ''' >>> import xml.etree.ElementTree as ET >>> tree = ET.fromstring(xml) + >>> parent_map = create_parent_map(tree) >>> ################ Missing type data ####################### - >>> print_var(tree,'::prop1',False,True,False,False,"short") + >>> print_var(tree,parent_map,'::prop1',False,True,False,False,"short") Traceback (most recent call last): SystemExit: ERROR: Error! Missing type information for prop1 - >>> print_var(tree,'prop2',True,False,False,False,"short") + >>> print_var(tree,parent_map,'prop2',True,False,False,False,"short") prop2 value: 2 type: integer valid values: ['1', '2'] - >>> print_var(tree,'prop2',False,True,False,False,"short") + >>> print_var(tree,parent_map,'prop2',False,True,False,False,"short") prop2: integer - >>> print_var(tree,'prop2',False,False,True,False,"short") + >>> print_var(tree,parent_map,'prop2',False,False,True,False,"short") 2 - >>> print_var(tree,'prop2',False,False,False,True,"short"," ") + >>> print_var(tree,parent_map,'prop2',False,False,False,True,"short"," ") prop2: ['1', '2'] """ expect (print_style in ["short","full"], f"Invalid print_style '{print_style}' for print_var. Use 'full' or 'short'.") - # Get node, along with all its parents (which might be used for 'full' print style) - node, parents = get_xml_node(xml_root,var) - # Get the shortest unique repr of the var name tokens = var.split("::") if tokens[0]=='': @@ -318,60 +474,33 @@ def print_var(xml_root,var,full,dtype,value,valid_values,print_style="invalid",i while len(tokens)>1: new_name = "::".join(tokens[1:]) - try: - get_xml_node(xml_root,new_name) - tokens.pop(0) - name = new_name - except AmbiguousName: - # new_name was either "" or an ambiguous name, and get_xml_node failed + matches = get_xml_nodes(xml_root, new_name) + if len(matches) > 1: break + else: + tokens.pop(0) - if print_style=="short": - # Just the inner most name - name = tokens[-1] - else: - name = "::".join(e.tag for e in parents) + "::" + node.tag + # Get node, along with all its parents (which might be used for 'full' print style) + matches = get_xml_nodes(xml_root,var) + expect(len(matches) == 1, f"Expected one match for {var}") + node = matches[0] - if full: - expect ("type" in node.attrib.keys(), - "Error! Missing type information for {}".format(name)) - print (f"{indent}{name}") - print (f"{indent} value: {node.text}") - print (f"{indent} type: {node.attrib['type']}") - if "valid_values" not in node.attrib.keys(): - valid = [] - else: - valid = node.attrib["valid_values"].split(",") - print (f"{indent} valid values: {valid}") - elif dtype: - expect ("type" in node.attrib.keys(), - "Error! Missing type information for {}".format(name)) - print (f"{indent}{name}: {node.attrib['type']}") - elif value: - print (f"{indent}{node.text}") - elif valid_values: - if "valid_values" not in node.attrib.keys(): - valid = '' - else: - valid = node.attrib["valid_values"].split(",") - print (f"{indent}{name}: {valid}") - else: - print (f"{indent}{name}: {node.text}") + print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) ############################################################################### -def print_all_vars(xml_root,xml_node,curr_namespace,full,dtype,value,valid_values,print_style,indent): +def print_all_vars(xml_root,xml_node,parent_map,curr_namespace,full,dtype,value,valid_values,print_style,indent): ############################################################################### print (f"{indent}{xml_node.tag}") for c in xml_node: if len(c)>0: - print_all_vars(xml_root,c,curr_namespace+c.tag+"::",full,dtype,value,valid_values,print_style,indent+" ") + print_all_vars(xml_root,c,parent_map,curr_namespace+c.tag+"::",full,dtype,value,valid_values,print_style,indent+" ") else: - print_var(xml_root,curr_namespace+c.tag,full,dtype,value,valid_values,print_style,indent+" ") + print_var(xml_root,parent_map,curr_namespace+c.tag,full,dtype,value,valid_values,print_style,indent+" ") ############################################################################### -def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, \ - dtype=False, valid_values=False): +def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, + dtype=False, valid_values=False, grep=False): ############################################################################### """ >>> xml = ''' @@ -386,21 +515,41 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, \ >>> import xml.etree.ElementTree as ET >>> tree = ET.fromstring(xml) >>> vars = ['prop2','::prop1'] - >>> success = atm_query_impl(tree, vars, False,False,False,False,False) + >>> success = atm_query_impl(tree, vars) root::sub::prop2: 2 root::prop1: one - >>> success = atm_query_impl(tree, [], True,False,False,False,True) + >>> success = atm_query_impl(tree, [], listall=True, valid_values=True) root prop1: sub prop1: prop2: ['1', '2'] + >>> success = atm_query_impl(tree,['prop1'], grep=True) + root::prop1: one + sub::prop1: two """ - + parent_map = create_parent_map(xml_root) if listall: - print_all_vars(xml_root,xml_root,"::",full,dtype,value,valid_values,"short"," ") + print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") + + elif grep: + for regex in variables: + expect("::" not in regex, "query --grep does not support including parent info") + var_re = re.compile(f'{regex}') + if var_re.search(xml_root.tag): + print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") + else: + for elem in xml_root: + if len(elem)>0: + atm_query_impl(elem,variables,listall,full,value,dtype,valid_values,grep) + else: + if var_re.search(elem.tag): + nodes = get_xml_nodes(xml_root, "::"+elem.tag) + expect(len(nodes) == 1, "No matches?") + print_var_impl(nodes[0],parent_map,full,dtype,value,valid_values,"full"," ") + else: for var in variables: - print_var(xml_root,var,full,dtype,value,valid_values,"full"," ") + print_var(xml_root,parent_map,var,full,dtype,value,valid_values,"full"," ") return True diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index f0611b484ca6..ff886ae83507 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -13,39 +13,71 @@ sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__ sys.path.append(os.path.dirname(os.path.realpath(__file__))) from eamxx_buildnml_impl import check_value, is_array_type -from atm_manip import get_xml_node, atm_config_chg_impl +from eamxx_buildnml import create_raw_xml_file +from atm_manip import atm_config_chg_impl, buffer_changes, reset_buffer, get_xml_nodes, parse_change from utils import run_cmd_no_fail, expect +# Add path to cime +_CIMEROOT = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","..","..","cime") +sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) +from standard_script_setup import * # pylint: disable=wildcard-import +from CIME.case import Case + ############################################################################### -def atm_config_chg(changes, no_buffer=False, reset=False): +def recreate_raw_xml_file(): ############################################################################### - expect(os.path.exists("namelist_scream.xml"), - "No pwd/namelist_scream.xml file is present. Please run from a case dir that has been set up") + caseroot = os.getcwd() + with Case(caseroot) as case: + create_raw_xml_file(case, caseroot) + +############################################################################### +def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): +############################################################################### + if not buffer_only: + expect(os.path.exists("namelist_scream.xml"), + "No pwd/namelist_scream.xml file is present. Please run from a case dir that has been set up") + else: + expect(not reset, "Makes no sense for buffer_only and reset to both be on") if reset: - run_cmd_no_fail("./xmlchange SCREAM_ATMCHANGE_BUFFER=''") - print("All buffered atmchanges have been removed. A fresh namelist_scream.xml will be generated the next time buildnml (case.setup) is run.") + reset_buffer() + print("All buffered atmchanges have been removed.") hack_xml = run_cmd_no_fail("./xmlquery SCREAM_HACK_XML --value") if hack_xml == "TRUE": print("SCREAM_HACK_XML is on. Removing namelist_scream.xml to force regen") os.remove("namelist_scream.xml") + recreate_raw_xml_file() return True else: expect(changes, "Missing = args") - with open("namelist_scream.xml", "r") as fd: - tree = ET.parse(fd) - root = tree.getroot() + # Before applying/buffering changes, at the very least check the syntax + for c in changes: + # This will throw if the syntax is bad + _, _, _ = parse_change(c) + + # If buffer_only=True, we must assume there were changes (we can't check). + # Otherwise, we'll assume no changes, and if we find one, we'll adjust + any_change = buffer_only + if not buffer_only: + with open("namelist_scream.xml", "r") as fd: + tree = ET.parse(fd) + root = tree.getroot() + + for change in changes: + this_changed = atm_config_chg_impl(root, change, all_matches) + any_change |= this_changed - any_change = atm_config_chg_impl(root,changes) if any_change: - tree.write("namelist_scream.xml") + # NOTE: if a change is wrong (e.g., typo in param name), we are still buffering it. + # We have no way of checking this, unfortunately. If you get an error that is + # not just syntax, your best course of action is to run atmchange --reset. + buffer_changes(changes, all_matches=all_matches) - if not no_buffer: - changes_str = " ".join(changes).replace(",",r"\,") - run_cmd_no_fail(f"./xmlchange --append SCREAM_ATMCHANGE_BUFFER='{changes_str}'") + if not buffer_only: + recreate_raw_xml_file() return True @@ -72,19 +104,26 @@ OR ) parser.add_argument( - "--no-buffer", + "-a", "--all", default=False, + dest="all_matches", action="store_true", - help="Used by buildnml to replay buffered commands", + help="Apply change to all entries matching the name" ) - parser.add_argument( - "--reset", + "-r", "--reset", default=False, action="store_true", help="Forget all previous atmchanges", ) + parser.add_argument( + "-b", "--buffer-only", + default=False, + action="store_true", + help="Only buffer the changes, don't actually do them. Useful for testmod scripts where the case is not setup yet", + ) + parser.add_argument("changes", nargs="*", help="Values to change") return parser.parse_args(args[1:]) diff --git a/components/eamxx/scripts/atmquery b/components/eamxx/scripts/atmquery index 7a0b8e07bdd6..f07ecefa94f1 100755 --- a/components/eamxx/scripts/atmquery +++ b/components/eamxx/scripts/atmquery @@ -13,11 +13,12 @@ sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__ sys.path.append(os.path.dirname(os.path.realpath(__file__))) from eamxx_buildnml_impl import check_value -from atm_manip import expect, get_xml_node, AmbiguousName, atm_query_impl +from atm_manip import atm_query_impl +from utils import expect ############################################################################### def atm_query(variables,listall=False,full=False,value=False, \ - dtype=False, valid_values=False): + dtype=False, valid_values=False, grep=False): ############################################################################### expect(os.path.exists("namelist_scream.xml"), "No pwd/namelist_scream.xml file is present. Please rum from a case dir that has been setup") @@ -26,29 +27,32 @@ def atm_query(variables,listall=False,full=False,value=False, \ tree = ET.parse(fd) xml_root = tree.getroot() - return atm_query_impl(xml_root,variables,listall,full,value,dtype,valid_values) + return atm_query_impl(xml_root,variables,listall,full,value,dtype,valid_values,grep) ############################################################################### def parse_command_line(args, description): ############################################################################### parser = argparse.ArgumentParser( - usage="""\n{0} [--listall] [--value] [--type] [--valid-values] [--full] [var1 [,var2 ...] + usage="""\n{0} [--grep] [--listall] [--value] [--type] [--valid-values] [--full] [var1 [,var2 ...] OR {0} --help \033[1mEXAMPLES:\033[0m - \033[1;32m# List all settings as VAR=VALUE + \033[1;32m# List all settings as VAR=VALUE\033[0m > {0} --listall - \033[1;32m# print var1 and var2 + \033[1;32m# print var1 and var2\033[0m > {0} var1 var2 - \033[1;32m# print var1 and var2, with full details + \033[1;32m# print var1 and var2, with full details\033[0m > {0} var1 var2 --full - \033[1;32m# print var1 type and valid values + \033[1;32m# print var1 type and valid values\033[0m > {0} var1 --type --valid-values + \033[1;32m# print all variables whose name matches expr\033[0m + > {0} --grep expr + """.format(pathlib.Path(args[0]).name), description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -62,16 +66,23 @@ OR ) parser.add_argument( - "--listall", + "--grep", default=False, action="store_true", + help="List all matching variables and their values.", + ) + + parser.add_argument( + "--listall", + action="store_true", help="List all variables and their values.", ) # The following options are mutually exclusive - group = parser.add_mutually_exclusive_group() + group1 = parser.add_argument_group(title="Display options") + group2 = parser.add_mutually_exclusive_group() - group.add_argument( + group2.add_argument( "--full", default=False, action="store_true", @@ -79,7 +90,7 @@ OR "valid values, description and file.", ) - group.add_argument( + group2.add_argument( "--value", default=False, action="store_true", @@ -87,14 +98,15 @@ OR "If more than one has been found print first value in list.", ) - group.add_argument( + group2.add_argument( "--type", default=False, + dest="dtype", action="store_true", help="Print the data type associated with each variable.", ) - group.add_argument( + group2.add_argument( "--valid-values", default=False, action="store_true", @@ -103,19 +115,16 @@ OR args = parser.parse_args(args[1:]) - if len(args.variables) == 1: - variables = args.variables[0].split(",") - else: - variables = args.variables - - return ( - variables, - args.listall, - args.full, - args.value, - args.type, - args.valid_values, - ) + if args.grep and args.listall: + parser.error("Cannot specify --listall and --grep at the same time") + + if args.grep and args.variables is None: + parser.error("Option --grep requires to pass a variable regex") + + if args.variables is not None and len(args.variables)==1: + args.variables = args.variables[0].split(",") + + return args ############################################################################### def _main_func(description): @@ -126,15 +135,7 @@ def _main_func(description): testmod() testmod(m=atm_manip) else: - ( - variables, - listall, - value, - full, - dtype, - valid_values, - ) = parse_command_line(sys.argv, description) - success = atm_query(variables,listall,value,full,dtype,valid_values) + success = atm_query(**vars(parse_command_line(sys.argv, description))) sys.exit(0 if success else 1) ############################################################################### diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 44c1a9d76579..0a60376bd45f 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -5,7 +5,8 @@ Script containing python test suite for SCREAM's CIME namelist-related infrastructure. """ -from utils import check_minimum_python_version, expect, ensure_pylint, run_cmd_assert_result, get_timestamp +from utils import check_minimum_python_version, expect, ensure_pylint, get_timestamp, \ + run_cmd_assert_result, run_cmd_no_fail, run_cmd check_minimum_python_version(3, 6) @@ -34,6 +35,8 @@ class TestBuildnml(unittest.TestCase): Convenience wrapper around create_test. Returns list of full paths to created cases. If multiple cases, the order of the returned list is not guaranteed to match the order of the arguments. """ + extra_args = extra_args.split() + test_id = f"cmd_nml_tests-{get_timestamp()}" extra_args.append("-t {}".format(test_id)) @@ -60,29 +63,61 @@ class TestBuildnml(unittest.TestCase): return cases[0] if len(cases) == 1 else cases ########################################################################### - def _chg_atmconfig(self, changes, case, buff=True, reset=False, expect_lost=False): + def _get_values(self, case, name, value=None, expect_equal=True, all_matches=False): ########################################################################### - buffer_opt = "" if buff else "--no-buffer" - - for name, value in changes: + """ + Queries a name, optionally checking if the value matches or does not match the + argument. + """ + if not all_matches: orig = run_cmd_assert_result(self, f"./atmquery {name} --value", from_dir=case) - self.assertNotEqual(orig, value) + if value: + if expect_equal: + self.assertEqual(orig, value, msg=name) + else: + self.assertNotEqual(orig, value, msg=name) + + return [name] + + else: + output = run_cmd_assert_result(self, f"./atmquery {name} --grep", from_dir=case).splitlines() + names = [line.rsplit(": ", maxsplit=1)[0].strip() for line in output] + values = [line.rsplit(": ", maxsplit=1)[1].strip() for line in output] + + if value: + for orig_value in values: + if expect_equal: + self.assertEqual(orig_value, value, msg=name) + else: + self.assertNotEqual(orig_value, value, msg=name) + + return names + + ########################################################################### + def _chg_atmconfig(self, changes, case, reset=False, expect_lost=None): + ########################################################################### + changes = [(item[0], item[1], False) if len(item) == 2 else item for item in changes] + + expect_lost = reset if expect_lost is None else expect_lost - run_cmd_assert_result(self, f"./atmchange {buffer_opt} {name}={value}", from_dir=case) - curr_value = run_cmd_assert_result(self, f"./atmquery {name} --value", from_dir=case) - self.assertEqual(curr_value, value) + changes_unpacked = {} + for name, value, all_matches in changes: + all_matches_opt = "-a" if all_matches else "" + + names = self._get_values(case, name, value=value, expect_equal=False, all_matches=all_matches) + + run_cmd_assert_result(self, f"./atmchange {all_matches_opt} {name}='{value}'", from_dir=case) + + for item in names: + changes_unpacked[item] = value if reset: run_cmd_assert_result(self, "./atmchange --reset", from_dir=case) - run_cmd_assert_result(self, "./case.setup", from_dir=case) + # run_cmd_assert_result(self, "./case.setup", from_dir=case) - for name, value in changes: - curr_value = run_cmd_assert_result(self, f"./atmquery {name} --value", from_dir=case) - if expect_lost: - self.assertNotEqual(curr_value, value) - else: - self.assertEqual(curr_value, value) + for name, value in changes_unpacked.items(): + self._get_values(case, name, value=value, expect_equal=not expect_lost) ########################################################################### def setUp(self): @@ -120,7 +155,7 @@ class TestBuildnml(unittest.TestCase): """ Test that xmlchanges impact atm config files """ - case = self._create_test("ERS_Ln22.ne30_ne30.F2010-SCREAMv1 --no-build".split()) + case = self._create_test("ERS_Ln22.ne30_ne30.F2010-SCREAMv1 --no-build") # atm config should match case test opts case_rest_n = run_cmd_assert_result(self, "./xmlquery REST_N --value", from_dir=case) @@ -143,67 +178,100 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are not lost when eamxx setup is called """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build".split()) + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + self._chg_atmconfig([("atm_log_level", "trace"), ("output_to_screen", "true")], case) - self._chg_atmconfig([("atm_log_level", "trace")], case) + run_cmd_no_fail ("./case.setup", from_dir=case) + + out1 = run_cmd_no_fail("./atmquery --value atm_log_level", from_dir=case) + out2 = run_cmd_no_fail("./atmquery --value output_to_screen", from_dir=case) + + expect (out1=="trace", "An atm change appears to have been lost during case.setup") + expect (out2=="true", "An atm change appears to have been lost during case.setup") ########################################################################### - def test_manual_atmchanges_are_lost(self): + def test_nml_defaults_append(self): ########################################################################### """ - Test that manual atmchanges are lost when eamxx setup is called + Test that the append attribute for array-type params in namelist defaults works as expected """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build".split()) + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + # Add testOnly proc + self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) - # An unbuffered atmchange is semantically the same as a manual edit - self._chg_atmconfig([("atm_log_level", "trace")], case, buff=False, expect_lost=True) + # Test case 1: append to base, then to last. Should give all 3 entries stacked + run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_VERTICAL_LEV 1'", from_dir=case) + run_cmd_no_fail("./case.setup", from_dir=case) + self._get_values(case,"my_param","1,2,3,4,5,6") + + # Test case 2: append to last, then to base. Should give 1st and 3rd entry + run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_VERTICAL_LEV 2'", from_dir=case) + run_cmd_no_fail("./case.setup", from_dir=case) + self._get_values(case,"my_param","1,2,5,6") ########################################################################### - def test_reset_atmchanges_are_lost(self): + def test_append(self): ########################################################################### """ - Test that manual atmchanges are lost when eamxx setup is called + Test that var+=value syntax behaves as expected """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build".split()) + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") - # An unbuffered atmchange is semantically the same as a manual edit - self._chg_atmconfig([("atm_log_level", "trace")], case, reset=True, expect_lost=True) + # Append to an existing entry + name = 'output_yaml_files' + out = run_cmd_no_fail(f"./atmchange {name}+=a.yaml", from_dir=case) + + # Get the yaml files + expected =f'{EAMXX_DIR / "data/scream_default_output.yaml"}, a.yaml' + self._get_values(case, name, value=expected, expect_equal=True) ########################################################################### - def test_manual_atmchanges_are_not_lost_hack_xml(self): + def test_reset_atmchanges_are_lost(self): ########################################################################### """ - Test that manual atmchanges are not lost when eamxx setup is called if - xml hacking is enabled. + Test that atmchanges are lost when resetting """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build".split()) - - run_cmd_assert_result(self, f"./xmlchange SCREAM_HACK_XML=TRUE", from_dir=case) + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") - self._chg_atmconfig([("atm_log_level", "trace")], case, buff=False) + self._chg_atmconfig([("atm_log_level", "trace")], case, reset=True) ########################################################################### - def test_multiple_atmchanges_are_preserved(self): + def test_atmchanges_are_lost_with_hack_xml(self): ########################################################################### """ - Test that multiple atmchanges are not lost when eamxx setup is called + Test that atmchanges are lost if SCREAM_HACK_XML=TRUE """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build".split()) + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") - self._chg_atmconfig([("atm_log_level", "trace"), ("output_to_screen", "true")], case) + run_cmd_assert_result(self, "./xmlchange SCREAM_HACK_XML=TRUE", from_dir=case) + + self._chg_atmconfig([("atm_log_level", "trace")], case, expect_lost=True) ########################################################################### def test_atmchanges_are_preserved_testmod(self): ########################################################################### """ - Test that atmchanges are not lost when eamxx setup is called when that - parameter is impacted by an active testmod + Test that atmchanges via testmod are preserved """ def_mach_comp = \ run_cmd_assert_result(self, "../CIME/Tools/list_e3sm_tests cime_tiny", from_dir=CIME_SCRIPTS_DIR).splitlines()[-1].split(".")[-1] - case = self._create_test(f"SMS.ne30_ne30.F2010-SCREAMv1.{def_mach_comp}.scream-scream_example_testmod_atmchange --no-build".split()) + case = self._create_test(f"SMS.ne30_ne30.F2010-SCREAMv1.{def_mach_comp}.scream-scream_example_testmod_atmchange --no-build") + + # Check that the value match what's in the testmod + out = run_cmd_no_fail("./atmquery --value cubed_sphere_map", from_dir=case) + expect (out=="42", "An atm change appears to have been lost during case.setup") + + ########################################################################### + def test_atmchanges_with_namespace(self): + ########################################################################### + """ + Test that atmchange works when using 'namespace' syntax foo::bar + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") - self._chg_atmconfig([("cubed_sphere_map", "84")], case) + self._chg_atmconfig([("p3::enable_precondition_checks", "false")], case) ########################################################################### def test_atmchanges_on_arrays(self): @@ -211,10 +279,116 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works for array data """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build".split()) + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") self._chg_atmconfig([("surf_mom_flux", "40.0,2.0")], case) + ########################################################################### + def test_atmchanges_on_all_matches(self): + ########################################################################### + """ + Test that atmchange --all works + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + self._chg_atmconfig([("enable_precondition_checks", "false", True)], case) + + ########################################################################### + def test_atmchanges_on_all_matches_plus_spec(self): + ########################################################################### + """ + Test atmchange --all followed by an atmchange of one of them + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + self._chg_atmconfig([("enable_precondition_checks", "false", True), + ("p3::enable_precondition_checks", "true")], case) + + ########################################################################### + def test_atmchanges_for_atm_procs_add(self): + ########################################################################### + """ + Test atmchanges that add atm procs + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) + + # If we are able to change subcycles of testOnly then we know the atmchange + # above added the necessary atm proc XML block. + self._chg_atmconfig([("testOnly::number_of_subcycles", "42")], case) + + ########################################################################### + def test_atmchanges_for_atm_procs_add_invalid(self): + ########################################################################### + """ + Test atmchanges that add atm procs + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + # modifying a procs list requires known processes or "_" pre/post suffixes + stat, out, err = run_cmd ("./atmchange mac_aero_mic::atm_procs_list=shoc,cldfraction,spa,p3,spiderman", + from_dir=case) + + expect (stat!=0,"Command './atmchange mac_aero_mic::atm_procs_list=shoc,cldFraction,spa,p3,spiderman' should have failed") + + ########################################################################### + def test_buffer_unchanged_with_bad_change_syntax(self): + ########################################################################### + """ + Test atmchange does not change buffer if syntax was wrong + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + # Attempting a bad change should not alter the content of SCREAM_ATMCHANGE_BUFFER + old = run_cmd_no_fail ("./xmlquery --value SCREAM_ATMCHANGE_BUFFER",from_dir=case) + stat, out, err = run_cmd ("./atmchange foo",from_dir=case) + expect (stat!=0,"Command './atmchange foo' should have failed") + + new = run_cmd_no_fail ("./xmlquery --value SCREAM_ATMCHANGE_BUFFER",from_dir=case) + + expect (new==old, "A bad atmchange should have not modified SCREAM_ATMCHANGE_BUFFER") + + ########################################################################### + def test_invalid_xml_option(self): + ########################################################################### + """ + Test atmchange errors out with invalid param names + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + stat, out, err = run_cmd ("./atmchange p3::non_existent=3",from_dir=case) + expect (stat!=0,"Command './atmchange p3::non_existent=3' should have failed") + + ########################################################################### + def test_atmchanges_for_atm_procs_add_group(self): + ########################################################################### + """ + Test atmchanges that add atm proc groups + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + out = run_cmd_no_fail ("./atmchange mac_aero_mic::atm_procs_list=shoc,_my_group_",from_dir=case) + + self._chg_atmconfig([("_my_group_::atm_procs_list", "testOnly")], case) + + # If we are able to change subcycles of testOnly then we know the atmchange + # above added the necessary atm proc XML block. + self._chg_atmconfig([("testOnly::number_of_subcycles", "42")], case) + + ########################################################################### + def test_atmchanges_for_atm_procs_remove(self): + ########################################################################### + """ + Test atmchanges that remove atm procs + """ + case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + + self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa")], case) + + stat, output, error = run_cmd("./atmquery --grep p3",from_dir=case) + expect (output=="", "There is still a trace of the removed process") + ############################################################################### def parse_command_line(args, desc): ############################################################################### diff --git a/components/eamxx/scripts/eamxx-params-docs-autogen b/components/eamxx/scripts/eamxx-params-docs-autogen new file mode 100755 index 000000000000..3024677d96b0 --- /dev/null +++ b/components/eamxx/scripts/eamxx-params-docs-autogen @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +""" +This script parses the file `cime_config/namelist_defaults_scream.xml' +and generates the markdown file `docs/common/eamxx_params.md`, +containing all the runtime parameters that can be configured via calls +to `atmchange` (in the case folder). For each parameter, we also report +a doc string and its type, as well as, if present, constraints and valid values. +""" + +import argparse, sys, os, pathlib + +from utils import _ensure_pylib_impl + +_ensure_pylib_impl("mdutils") + +import xml.etree.ElementTree as ET +from mdutils.mdutils import MdUtils +from mdutils import Html + +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "cime_config")) +from eamxx_buildnml_impl import resolve_all_inheritances, get_valid_selectors + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""{0} +""".format(pathlib.Path(args[0]).name), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + return parser.parse_args(args[1:]) + +########################################################################### +def add_param(docs,scope,item): +########################################################################### + # Locked parameters are not to be configured at runtime, so don't even bother + # E.g, a locked param is something we need to get in the input file, like + # the restart write frequency, but we don't want the user to modify it + # via atmchange + if "locked" in item.attrib.keys(): + return + docs.new_line(f"* {scope}{item.tag}:") + + pdoc = item.attrib['doc'] if 'doc' in item.attrib.keys() else "**MISSING**" + docs.new_line(f" - description: {pdoc}") + + ptype = item.attrib['type'] if 'type' in item.attrib.keys() else "**MISSING**" + docs.new_line(f" - type: {ptype}") + + pvalid = item.attrib['valid_values'] if 'valid_values' in item.attrib.keys() else None + if pvalid is not None: + docs.new_line(f" - valid values: {pvalid}") + pconstr = item.attrib['constraints'] if 'constraints' in item.attrib.keys() else None + if pconstr is not None: + docs.new_line(f" - constraints: {pconstr}") + +########################################################################### +def add_children(docs,xml,scope=""): +########################################################################### + done = [] + # Locked parameters are not to be configured at runtime, so don't even bother + # E.g, a locked param is something we need to get in the input file, like + # the restart write frequency, but we don't want the user to modify it + # via atmchange + if "locked" in xml.attrib.keys(): + return + for item in xml: + # The same entry may appear multiple times in the XML defaults file, + # each time with different selectors. We don't want to generate the + # same documentation twice. + if item.tag in done: + continue + done.append(item.tag) + if len(item)>0: + add_children (docs,item,f"{scope}{xml.tag}::") + else: + add_param(docs,f"{scope}{xml.tag}::",item) + docs.new_line() + +########################################################################### +def generate_params_docs(): +########################################################################### + + eamxx = pathlib.Path(__file__).parent.parent.resolve() + xml_defaults_file = eamxx / "cime_config" / "namelist_defaults_scream.xml" + output_file = eamxx / "docs" / "common" / "eamxx_params.md" + + print("Generating eamxx params documentation...") + print(f" output file: {output_file}") + + with open(xml_defaults_file, "r") as fd: + tree = ET.parse(fd) + xml_defaults = tree.getroot() + + selectors = get_valid_selectors(xml_defaults) + resolve_all_inheritances(xml_defaults) + + docs = MdUtils(file_name=str(output_file),title='EAMxx runtime configurable parameters') + with open (output_file, "w") as fd: + docs.new_header(level=1,title='Atmosphere Processes Parameters') + aps = xml_defaults.find('atmosphere_processes_defaults') + for ap in aps: + if ap.tag.startswith('atm_proc'): + continue + docs.new_header(level=2,title=ap.tag) + add_children(docs,ap) + + ic = xml_defaults.find('initial_conditions') + docs.new_header(level=1,title="Initial Conditions Parameters") + add_children(docs,ic) + + ad = xml_defaults.find('driver_options') + docs.new_header(level=1,title='Atmosphere Driver Parameters') + add_children(docs,ad) + + scorpio = xml_defaults.find('Scorpio') + docs.new_header(level=1,title='Scorpio Parameters') + add_children(docs,scorpio) + + homme = xml_defaults.find('ctl_nl') + docs.new_header(level=1,title='Homme namelist') + add_children(docs,homme) + docs.create_md_file() + + print("Generating eamxx params documentation ... SUCCESS!") + return True + +############################################################################### +def _main_func(description): +############################################################################### + + success = generate_params_docs(**vars(parse_command_line(sys.argv, description))) + + sys.exit(0 if success else 1) + +############################################################################### + +if (__name__ == "__main__"): + _main_func(__doc__) diff --git a/components/eamxx/scripts/gather_all_data.py b/components/eamxx/scripts/gather_all_data.py index 6a6417b373a0..9c9b1726c465 100644 --- a/components/eamxx/scripts/gather_all_data.py +++ b/components/eamxx/scripts/gather_all_data.py @@ -98,6 +98,7 @@ def run_on_machine(self, machine): if self._local: run_cmd_no_fail(cmd, arg_stdout=None, arg_stderr=None, verbose=True, dry_run=self._dry_run, exc_type=RuntimeError) else: + output = "" # Making pylint happy try: ssh_cmd = "ssh -o StrictHostKeyChecking=no {} '{}'".format(machine, cmd) output = run_cmd_no_fail(ssh_cmd, dry_run=self._dry_run, exc_type=RuntimeError, combine_output=True) diff --git a/components/eamxx/scripts/gen_boiler.py b/components/eamxx/scripts/gen_boiler.py index 95838a7c631a..d45197deeeba 100644 --- a/components/eamxx/scripts/gen_boiler.py +++ b/components/eamxx/scripts/gen_boiler.py @@ -2,7 +2,8 @@ from git_utils import get_git_toplevel_dir from collections import OrderedDict -import pathlib, re, os +import re +from pathlib import Path # # Global hardcoded data @@ -11,67 +12,67 @@ # Templates: maps piece name to generic file text FILE_TEMPLATES = { "cxx_bfb_unit_impl": lambda phys, sub, gen_code: -"""#include "catch2/catch.hpp" +f"""#include "catch2/catch.hpp" #include "share/scream_types.hpp" #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "physics/{physics}/{physics}_functions.hpp" -#include "physics/{physics}/{physics}_functions_f90.hpp" +#include "physics/{phys}/{phys}_functions.hpp" +#include "physics/{phys}/{phys}_functions_f90.hpp" -#include "{physics}_unit_tests_common.hpp" +#include "{phys}_unit_tests_common.hpp" namespace scream {{ -namespace {physics} {{ +namespace {phys} {{ namespace unit_test {{ template -struct UnitWrap::UnitTest::{test_data_struct} {{ +struct UnitWrap::UnitTest::{get_data_test_struct_name(sub)} {{ {gen_code} }}; }} // namespace unit_test -}} // namespace {physics} +}} // namespace {phys} }} // namespace scream namespace {{ -TEST_CASE("{sub}_bfb", "[{physics}]") +TEST_CASE("{sub}_bfb", "[{phys}]") {{ - using TestStruct = scream::{physics}::unit_test::UnitWrap::UnitTest::{test_data_struct}; + using TestStruct = scream::{phys}::unit_test::UnitWrap::UnitTest::{get_data_test_struct_name(sub)}; TestStruct::run_bfb(); }} }} // empty namespace -""".format(physics=phys, sub=sub, test_data_struct=get_data_test_struct_name(sub), gen_code=gen_code), +""", ############################################################################### "cxx_func_impl": lambda phys, sub, gen_code: -"""#ifndef {phys_upper}_{sub_upper}_IMPL_HPP -#define {phys_upper}_{sub_upper}_IMPL_HPP +f"""#ifndef {phys.upper()}_{sub.upper()}_IMPL_HPP +#define {phys.upper()}_{sub.upper()}_IMPL_HPP -#include "{physics}_functions.hpp" // for ETI only but harmless for GPU +#include "{phys}_functions.hpp" // for ETI only but harmless for GPU namespace scream {{ -namespace {physics} {{ +namespace {phys} {{ /* - * Implementation of {physics} {sub}. Clients should NOT - * #include this file, but include {physics}_functions.hpp instead. + * Implementation of {phys} {sub}. Clients should NOT + * #include this file, but include {phys}_functions.hpp instead. */ template {gen_code} -}} // namespace {physics} +}} // namespace {phys} }} // namespace scream #endif -""".format(physics=phys, sub=sub, gen_code=gen_code, phys_upper=phys.upper(), sub_upper=sub.upper()), +""", ############################################################################### @@ -83,16 +84,16 @@ FILEPATH, FILECREATE, INSERT_REGEX, ID_SELF_BEGIN_REGEX, ID_SELF_END_REGEX, DESC = range(6) PIECES = OrderedDict([ ("f90_c2f_bind", ( - lambda phys, sub, gb: "{}_iso_c.f90".format(phys), + lambda phys, sub, gb: f"{phys}_iso_c.f90", lambda phys, sub, gb: expect_exists(phys, sub, gb, "f90_c2f_bind"), - lambda phys, sub, gb: re.compile(r"^\s*end\s+module\s{}_iso_c".format(phys)), # put at end of module + lambda phys, sub, gb: re.compile(fr"^\s*end\s+module\s{phys}_iso_c"), # put at end of module lambda phys, sub, gb: get_subroutine_begin_regex(sub + "_c"), # sub_c begin lambda phys, sub, gb: get_subroutine_end_regex(sub + "_c"), # sub_c end lambda *x : "The c to f90 fortran subroutine(_c)" )), ("f90_f2c_bind" , ( - lambda phys, sub, gb: "{}_iso_f.f90".format(phys), + lambda phys, sub, gb: f"{phys}_iso_f.f90", lambda phys, sub, gb: expect_exists(phys, sub, gb, "f90_f2c_bind"), lambda phys, sub, gb: re.compile(r"^\s*end\s+interface"), # put at end of interface lambda phys, sub, gb: get_subroutine_begin_regex(sub + "_f"), # sub_f begin @@ -101,7 +102,7 @@ )), ("cxx_c2f_bind_decl" , ( - lambda phys, sub, gb: "{}_functions_f90.cpp".format(phys), + lambda phys, sub, gb: f"{phys}_functions_f90.cpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_c2f_bind_decl"), lambda phys, sub, gb: get_cxx_close_block_regex(comment='extern "C" : end _c decls'), # reqs special comment lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_c"), # cxx_c decl @@ -110,7 +111,7 @@ )), ("cxx_c2f_glue_decl" , ( - lambda phys, sub, gb: "{}_functions_f90.hpp".format(phys), + lambda phys, sub, gb: f"{phys}_functions_f90.hpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_c2f_glue_decl"), lambda phys, sub, gb: re.compile(r'^\s*extern\s+"C"'), # put before _f decls lambda phys, sub, gb: get_cxx_function_begin_regex(sub), # cxx(data) decl @@ -119,7 +120,7 @@ )), ("cxx_c2f_glue_impl" , ( - lambda phys, sub, gb: "{}_functions_f90.cpp".format(phys), + lambda phys, sub, gb: f"{phys}_functions_f90.cpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_c2f_glue_impl"), lambda phys, sub, gb: re.compile(r"^\s*// end _c impls"), # reqs special comment lambda phys, sub, gb: get_cxx_function_begin_regex(sub), # cxx(data) @@ -128,7 +129,7 @@ )), ("cxx_c2f_data" , ( - lambda phys, sub, gb: "{}_functions_f90.hpp".format(phys), + lambda phys, sub, gb: f"{phys}_functions_f90.hpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_c2f_data"), lambda phys, sub, gb: re.compile(r"^\s*// Glue functions to call fortran"), # reqs special comment lambda phys, sub, gb: get_cxx_struct_begin_regex(get_data_struct_name(sub)), # struct Sub @@ -137,7 +138,7 @@ )), ("cxx_f2c_bind_decl" , ( - lambda phys, sub, gb: "{}_functions_f90.hpp".format(phys), + lambda phys, sub, gb: f"{phys}_functions_f90.hpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_f2c_bind_decl"), lambda phys, sub, gb: get_cxx_close_block_regex(comment="end _f function decls"), # reqs special comment lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_f"), # cxx_f decl @@ -146,7 +147,7 @@ )), ("cxx_f2c_bind_impl" , ( - lambda phys, sub, gb: "{}_functions_f90.cpp".format(phys), + lambda phys, sub, gb: f"{phys}_functions_f90.cpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_f2c_bind_impl"), lambda phys, sub, gb: get_namespace_close_regex(phys), # insert at end of namespace lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_f"), # cxx_f @@ -155,7 +156,7 @@ )), ("cxx_func_decl", ( - lambda phys, sub, gb: "{}_functions.hpp".format(phys), + lambda phys, sub, gb: f"{phys}_functions.hpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_func_decl"), lambda phys, sub, gb: get_cxx_close_block_regex(semicolon=True, comment="struct Functions"), # end of struct, reqs special comment lambda phys, sub, gb: get_cxx_function_begin_regex(sub, static=True), # cxx decl @@ -164,16 +165,16 @@ )), ("cxx_incl_impl", ( - lambda phys, sub, gb: "{}_functions.hpp".format(phys), + lambda phys, sub, gb: f"{phys}_functions.hpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_incl_impl"), - lambda phys, sub, gb: re.compile(r"^\s*#\s*endif\s+//\s*KOKKOS_ENABLE_CUDA"), # insert at end of impl includes, reqs special comment + lambda phys, sub, gb: re.compile(r"^\s*#\s*endif\s+//\s*GPU"), # insert at end of impl includes, reqs special comment lambda phys, sub, gb: re.compile(r'^\s*#\s*include\s+"{}"'.format(get_piece_data(phys, sub, "cxx_func_impl", FILEPATH, gb))), lambda phys, sub, gb: re.compile(r".*"), lambda *x : "The include of *impl.hpp file at bottom of main hpp" )), ("cxx_func_impl", ( - lambda phys, sub, gb: "{}_{}_impl.hpp".format(phys, sub), + lambda phys, sub, gb: f"impl/{phys}_{sub}_impl.hpp", lambda phys, sub, gb: create_template(phys, sub, gb, "cxx_func_impl"), lambda phys, sub, gb: get_namespace_close_regex(phys), # insert at end of namespace lambda phys, sub, gb: get_cxx_function_begin_regex(sub, template="Functions"), # cxx begin @@ -182,7 +183,7 @@ )), ("cxx_bfb_unit_decl", ( - lambda phys, sub, gb: "tests/{}_unit_tests_common.hpp".format(phys), + lambda phys, sub, gb: f"tests/{phys}_unit_tests_common.hpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_bfb_unit_decl"), lambda phys, sub, gb: get_cxx_close_block_regex(semicolon=True), # insert at end of test struct lambda phys, sub, gb: get_cxx_struct_begin_regex(get_data_test_struct_name(sub)), # struct decl @@ -191,7 +192,7 @@ )), ("cxx_bfb_unit_impl", ( - lambda phys, sub, gb: "tests/{}_{}_tests.cpp".format(phys, sub), + lambda phys, sub, gb: f"tests/{phys}_{sub}_tests.cpp", lambda phys, sub, gb: create_template(phys, sub, gb, "cxx_bfb_unit_impl"), lambda phys, sub, gb: get_cxx_close_block_regex(semicolon=True, at_line_start=True), # insert of end of struct lambda phys, sub, gb: get_cxx_function_begin_regex("run_bfb", static=True), # run_bfb @@ -200,7 +201,7 @@ )), ("cxx_eti", ( - lambda phys, sub, gb: "{}_{}.cpp".format(phys, sub), + lambda phys, sub, gb: f"eti/{phys}_{sub}.cpp", lambda phys, sub, gb: create_template(phys, sub, gb, "cxx_eti"), lambda phys, sub, gb: re.compile(".*"), # insert at top of file lambda phys, sub, gb: re.compile(".*"), # start at top of file @@ -211,8 +212,8 @@ ("cmake_impl_eti", ( lambda phys, sub, gb: "CMakeLists.txt", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cmake_impl_eti"), - lambda phys, sub, gb: re.compile(r".*[)]\s*#\s*{} ETI SRCS".format(phys.upper())), # insert at end of ETI src list, reqs special comment - lambda phys, sub, gb: re.compile(r".*{}".format(get_piece_data(phys, sub, "cxx_eti", FILEPATH, gb))), + lambda phys, sub, gb: re.compile(fr".*[)]\s*#\s*{phys.upper()} ETI SRCS"), # insert at end of ETI src list, reqs special comment + lambda phys, sub, gb: re.compile(fr".*{get_piece_data(phys, sub, 'cxx_eti', FILEPATH, gb)}"), lambda phys, sub, gb: re.compile(".*"), lambda *x : "Make cmake aware of the ETI file if not cuda build" )), @@ -220,8 +221,8 @@ ("cmake_unit_test", ( lambda phys, sub, gb: "tests/CMakeLists.txt", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cmake_unit_test"), - lambda phys, sub, gb: re.compile(r".*[)]\s*#\s*{}_TESTS_SRCS".format(phys.upper())), # insert at end of test src list, reqs special comment - lambda phys, sub, gb: re.compile(r".*{}".format(os.path.basename(get_piece_data(phys, sub, "cxx_bfb_unit_impl", FILEPATH, gb)))), + lambda phys, sub, gb: re.compile(fr".*[)]\s*#\s*{phys.upper()}_TESTS_SRCS"), # insert at end of test src list, reqs special comment + lambda phys, sub, gb: re.compile(fr".*{Path(get_piece_data(phys, sub, 'cxx_bfb_unit_impl', FILEPATH, gb)).name}"), lambda phys, sub, gb: re.compile(".*"), lambda *x : "Make cmake aware of the unit test" )), @@ -230,17 +231,22 @@ # physics map. maps the name of a physics packages containing the original fortran subroutines to: # (path-to-origin, path-to-cxx-src) -ORIGIN_FILE, CXX_ROOT, INIT_CODE = range(3) +ORIGIN_FILES, CXX_ROOT, INIT_CODE = range(3) PHYSICS = { "p3" : ( - "components/eam/src/physics/cam/micro_p3.F90", + ("components/eam/src/physics/cam/micro_p3.F90",), "components/eamxx/src/physics/p3", "p3_init();" ), "shoc" : ( - "components/eam/src/physics/cam/shoc.F90", + ("components/eam/src/physics/cam/shoc.F90",), "components/eamxx/src/physics/shoc", - "shoc_init(REPLACE_ME, true);" + "shoc_init(d.nlev, true);" + ), + "dp" : ( + ("components/eam/src/control/apply_iop_forcing.F90", "components/eam/src/dynamics/se/se_iop_intr_mod.F90", "components/eam/src/control/iop_data_mod.F90", "components/eam/src/control/history_iop.F90"), + "components/eamxx/src/physics/dp", + "dp_init(d.plev, true);" ), } @@ -316,7 +322,7 @@ def get_subroutine_begin_regex(name): >>> bool(get_subroutine_begin_regex("fake_sub").match("subroutine fake_sub")) False """ - subroutine_begin_regex_str = r"^\s*subroutine\s+{}\s*[(]".format(name) + subroutine_begin_regex_str = fr"^\s*subroutine\s+{name}\s*[(]" return re.compile(subroutine_begin_regex_str) ############################################################################### @@ -338,7 +344,7 @@ def get_function_begin_regex(name): >>> bool(get_function_begin_regex("fake_sub").match("end function fake_sub")) False """ - function_begin_regex_str = r"^\s*((pure\s+)?function)\s+{}\s*[(].*result\s*[(]\s*([^) ]+)".format(name) + function_begin_regex_str = fr"^\s*((pure\s+)?function)\s+{name}\s*[(].*result\s*[(]\s*([^) ]+)" return re.compile(function_begin_regex_str) ############################################################################### @@ -358,7 +364,7 @@ def get_subroutine_end_regex(name): >>> bool(get_subroutine_end_regex("fake_sub").match("end function fake_sub_2")) False """ - subroutine_end_regex_str = r"^\s*end\s+(subroutine|function)\s+{}\s*$".format(name) + subroutine_end_regex_str = fr"^\s*end\s+(subroutine|function)\s+{name}\s*$" return re.compile(subroutine_end_regex_str) ############################################################################### @@ -383,8 +389,8 @@ def get_cxx_function_begin_regex(name, static=False, template=None): True """ static_regex_str = r"static\s+" if static else "" - template_regex_str = r"{}::".format(template) if template else "" - function_begin_regex_str = r"^\s*{}void\s+{}{}\s*[(]".format(static_regex_str, template_regex_str, name) + template_regex_str = fr"{template}::" if template else "" + function_begin_regex_str = fr"^\s*{static_regex_str}void\s+{template_regex_str}{name}\s*[(]" return re.compile(function_begin_regex_str) ############################################################################### @@ -420,8 +426,8 @@ def get_cxx_close_block_regex(semicolon=False, comment=None, at_line_start=False """ semicolon_regex_str = r"\s*;" if semicolon else "" line_start_regex_str = "" if at_line_start else r"\s*" - comment_regex_str = r"\s*//\s*{}".format(comment) if comment else "" - close_block_regex_str = re.compile(r"^{}}}{}{}\s*$".format(line_start_regex_str, semicolon_regex_str, comment_regex_str)) + comment_regex_str = fr"\s*//\s*{comment}" if comment else "" + close_block_regex_str = re.compile(fr"^{line_start_regex_str}}}{semicolon_regex_str}{comment_regex_str}\s*$") return re.compile(close_block_regex_str) ############################################################################### @@ -433,7 +439,7 @@ def get_namespace_close_regex(namespace): >>> bool(get_namespace_close_regex("foo").match(" } // namespace foo_bar")) False """ - return get_cxx_close_block_regex(comment=r"namespace\s+{}".format(namespace)) + return get_cxx_close_block_regex(comment=fr"namespace\s+{namespace}") ############################################################################### def get_cxx_struct_begin_regex(struct): @@ -446,7 +452,7 @@ def get_cxx_struct_begin_regex(struct): >>> bool(get_cxx_struct_begin_regex("Foo").match("struct FooBar")) False """ - struct_regex_str = r"^\s*struct\s+{}([\W]|$)".format(struct) + struct_regex_str = fr"^\s*struct\s+{struct}([\W]|$)" return re.compile(struct_regex_str) ############################################################################### @@ -469,7 +475,7 @@ def get_data_test_struct_name(sub): >>> get_data_test_struct_name("update_prognostics_implicit") 'TestUpdatePrognosticsImplicit' """ - return "Test{}".format(get_data_struct_name(sub)[:-4]) + return f"Test{get_data_struct_name(sub)[:-4]}" ############################################################################### def get_supported_pieces(): @@ -495,8 +501,8 @@ def get_physics_data(physics_name, physics_data): def expect_exists(physics, sub, gb, piece): ############################################################################### filepath = gb.get_path_for_piece_file(physics, sub, piece) - expect(filepath.exists(), "For generating {}'s {} for phyiscs {}, expected file {} to already exist".\ - format(sub, piece, physics, filepath)) + expect(filepath.exists(), + f"For generating {sub}'s {piece} for phyiscs {physics}, expected file {filepath} to already exist") return False # File was not created ############################################################################### @@ -507,7 +513,7 @@ def create_template(physics, sub, gb, piece, force=False, force_arg_data=None): >>> gb = GenBoiler(["linear_interp"], ["cxx_func_impl"], dry_run=True) >>> create_template("shoc", "linear_interp", gb, "cxx_func_impl", force=True, force_arg_data=UT_ARG_DATA) #doctest: +ELLIPSIS - Would create file .../components/eamxx/src/physics/shoc/shoc_linear_interp_impl.hpp with contents: + Would create file .../components/eamxx/src/physics/shoc/impl/shoc_linear_interp_impl.hpp with contents: #ifndef SHOC_LINEAR_INTERP_IMPL_HPP #define SHOC_LINEAR_INTERP_IMPL_HPP @@ -539,12 +545,12 @@ def create_template(physics, sub, gb, piece, force=False, force_arg_data=None): filepath = gb.get_path_for_piece_file(physics, sub, piece) if not filepath.exists() or force: expect(piece in FILE_TEMPLATES, - "{} does not exist and there is no template for generating files for piece {}".format(filepath, piece)) + f"{filepath} does not exist and there is no template for generating files for piece {piece}") - gen_code = getattr(gb, "gen_{}".format(piece))(physics, sub, force_arg_data=force_arg_data) + gen_code = getattr(gb, f"gen_{piece}")(physics, sub, force_arg_data=force_arg_data) contents = FILE_TEMPLATES[piece](physics, sub, gen_code) if gb.dry_run(): - print("Would create file {} with contents:\n{}".format(filepath, contents)) + print(f"Would create file {filepath} with contents:\n{contents}") else: with filepath.open("w", encoding="utf-8") as fd: fd.write(contents) @@ -662,7 +668,7 @@ def split_top_commas(line): if balanced: top_splits.append(raw_split) else: - top_splits[-1] += ",{}".format(raw_split) + top_splits[-1] += f",{raw_split}" balanced = top_splits[-1].count("(") == top_splits[-1].count(")") @@ -689,7 +695,7 @@ def get_arg_order(line): first_paren_contents = "" for c in line: if c == "(": - expect(not first_paren, "Bad line, multiple opening parens: {}".format(line)) + expect(not first_paren, f"Bad line, multiple opening parens: {line}") first_paren = True elif c == ")": break @@ -733,20 +739,31 @@ def parse_f90_args(line): [('x1', 'real', 'in', ('ncol', 'km1')), ('y1', 'real', 'in', ('ncol', 'km1'))] >>> parse_f90_args('real(rtype), intent(in) :: x1(ncol,km1,ntracers)') [('x1', 'real', 'in', ('ncol', 'km1', 'ntracers'))] + >>> parse_f90_args('type(element_t), intent(inout) :: elem(:)') + [('elem', 'type::element_t', 'inout', (':',))] + >>> parse_f90_args('character*(max_path_len), intent(out), optional :: iopfile_out') + [('iopfile_out', 'type::string', 'out', None)] """ - expect(line.count("::") == 1, "Expected line format 'type-info :: names' for: {}".format(line)) + expect(line.count("::") == 1, f"Expected line format 'type-info :: names' for: {line}") metadata_str, names_str = line.split("::") names_dims = split_top_commas(names_str) metadata = split_top_commas(metadata_str) - argtype = metadata[0].split("(")[0].strip() + argtoken = metadata[0] + argtype = argtoken.split("(")[0].strip() + if argtype == "type": + expect("(" in argtoken, f"Undefined type for {argtoken}") + argtype += ("::" + argtoken.split("(")[1].strip().rstrip(")")) + elif argtype == "character*": + argtype = "type::string" + intent, dims = None, None for metadatum in metadata: if metadatum.startswith("intent"): - expect(intent is None, "Multiple intents in line: {}".format(line)) + expect(intent is None, f"Multiple intents in line: {line}") intent = metadatum.split("(")[-1].rstrip(")").strip() elif metadatum.startswith("dimension"): - expect(dims is None, "Multiple dimensions in line: {}".format(line)) + expect(dims is None, f"Multiple dimensions in line: {line}") dims_raw = metadatum.split("(")[-1].rstrip(")").strip() dims = tuple(item.replace(" ", "") for item in dims_raw.split(",")) @@ -756,7 +773,7 @@ def parse_f90_args(line): name, dims_raw = name_dim.split("(") dims_raw = dims_raw.rstrip(")").strip() dims_check = tuple(item.replace(" ", "") for item in dims_raw.split(",")) - expect(dims is None or dims_check == dims, "Inconsistent dimensions in line: {}".format(line)) + expect(dims is None or dims_check == dims, f"Inconsistent dimensions in line: {line}") dims = dims_check names.append(name.strip()) else: @@ -833,12 +850,41 @@ def parse_origin(contents, subs): ... ... return foo ... end function impli_srf_stress_term + ... + ... subroutine advance_iop_forcing(scm_dt, ps_in, & ! In + ... u_in, v_in, t_in, q_in, t_phys_frc,& ! In + ... u_update, v_update, t_update, q_update) ! Out + ... + ... ! Input arguments + ... real(r8), intent(in) :: ps_in ! surface pressure [Pa] + ... real(r8), intent(in) :: u_in(plev) ! zonal wind [m/s] + ... real(r8), intent(in) :: v_in(plev) ! meridional wind [m/s] + ... real(r8), intent(in) :: t_in(plev) ! temperature [K] + ... real(r8), intent(in) :: q_in(plev,pcnst) ! q tracer array [units vary] + ... real(r8), intent(in) :: t_phys_frc(plev) ! temperature forcing from physics [K/s] + ... real(r8), intent(in) :: scm_dt ! model time step [s] + ... + ... ! Output arguments + ... real(r8), intent(out) :: t_update(plev) ! updated temperature [K] + ... real(r8), intent(out) :: q_update(plev,pcnst)! updated q tracer array [units vary] + ... real(r8), intent(out) :: u_update(plev) ! updated zonal wind [m/s] + ... real(r8), intent(out) :: v_update(plev) ! updated meridional wind [m/s] + ... + ... end subroutine advance_iop_forcing + ... + ... subroutine iop_setinitial(elem) + ... type(element_t), intent(inout) :: elem(:) + ... end subroutine iop_setinitial ... ''' >>> print("\n".join([str(item) for item in sorted(parse_origin(teststr, ["p3_get_tables", "p3_init_b"]).items())])) ('p3_get_tables', [('mu_r_user', 'real', 'out', ('150',)), ('revap_user', 'real', 'out', ('300', '10')), ('tracerd', 'real', 'out', ('300', '10', '42')), ('vn_user', 'real', 'out', ('300', '10')), ('vm_user', 'real', 'out', ('300', '10'))]) ('p3_init_b', []) >>> print("\n".join([str(item) for item in parse_origin(teststr, ["impli_srf_stress_term"]).items()])) ('impli_srf_stress_term', [('shcol', 'integer', 'in', None), ('rho_zi_sfc', 'real', 'in', ('shcol',)), ('uw_sfc', 'real', 'in', ('shcol',)), ('vw_sfc', 'real', 'in', ('shcol',)), ('u_wind_sfc', 'real', 'in', ('shcol',)), ('v_wind_sfc', 'real', 'in', ('shcol',)), ('ksrf', 'real', 'out', ('shcol',))]) + >>> print("\n".join([str(item) for item in parse_origin(teststr, ["advance_iop_forcing"]).items()])) + ('advance_iop_forcing', [('plev', 'integer', 'in', None), ('pcnst', 'integer', 'in', None), ('scm_dt', 'real', 'in', None), ('ps_in', 'real', 'in', None), ('u_in', 'real', 'in', ('plev',)), ('v_in', 'real', 'in', ('plev',)), ('t_in', 'real', 'in', ('plev',)), ('q_in', 'real', 'in', ('plev', 'pcnst')), ('t_phys_frc', 'real', 'in', ('plev',)), ('u_update', 'real', 'out', ('plev',)), ('v_update', 'real', 'out', ('plev',)), ('t_update', 'real', 'out', ('plev',)), ('q_update', 'real', 'out', ('plev', 'pcnst'))]) + >>> print("\n".join([str(item) for item in parse_origin(teststr, ["iop_setinitial"]).items()])) + ('iop_setinitial', [('elem', 'type::element_t', 'inout', (':',))]) """ begin_sub_regexes = [get_subroutine_begin_regex(sub) for sub in subs] begin_func_regexes = [get_function_begin_regex(sub) for sub in subs] @@ -856,11 +902,11 @@ def parse_origin(contents, subs): begin_sub_match = begin_sub_regex.match(line) begin_func_match = begin_func_regex.match(line) if begin_sub_match is not None: - expect(active_sub is None, "subroutine {} was still active when {} began".format(active_sub, sub)) + expect(active_sub is None, f"subroutine {active_sub} was still active when {sub} began") active_sub = sub arg_order = get_arg_order(line) elif begin_func_match is not None: - expect(active_sub is None, "subroutine {} was still active when {} began".format(active_sub, sub)) + expect(active_sub is None, f"subroutine {active_sub} was still active when {sub} began") active_sub = sub arg_order = get_arg_order(line) result_name = begin_func_match.groups()[-1] @@ -870,7 +916,7 @@ def parse_origin(contents, subs): if decl_match is not None: arg_decls.extend(parse_f90_args(line)) elif result_name: - result_decl_regex = re.compile(r".+::\s*{}([^\w]|$)".format(result_name)) + result_decl_regex = re.compile(fr".+::\s*{result_name}([^\w]|$)") result_decl_match = result_decl_regex.match(line) if result_decl_match is not None: line = line.replace("::", " , intent(out) ::") @@ -879,9 +925,9 @@ def parse_origin(contents, subs): end_regex = get_subroutine_end_regex(active_sub) end_match = end_regex.match(line) if end_match is not None: - expect(active_sub not in db, "Found multiple matches for {}".format(active_sub)) + expect(active_sub not in db, f"Found multiple matches for {active_sub}") expect(len(arg_order) == len(arg_decls), - "Number of decls:\n{}\nDid not match arg list: {}".format(arg_decls, arg_order)) + f"Number of decls:\n{arg_decls}\nDid not match arg list: {arg_order}") # we need our decls to be ordered based on arg list order ordered_decls = [] @@ -893,9 +939,25 @@ def parse_origin(contents, subs): found = True break - expect(found, "Could not find decl for arg {} in\n{}".format(arg, arg_decls)) - - db[active_sub] = ordered_decls + expect(found, f"Could not find decl for arg {arg} in\n{arg_decls}") + + # Dim resolution. Arrays with global dims must have the + # dim as an input in the converted code. + global_ints_to_insert = [] + arg_names = set() + for arg_datum in ordered_decls: + arg_name = arg_datum[ARG_NAME] + arg_names.add(arg_name) + + for arg_datum in ordered_decls: + arg_dims = arg_datum[ARG_DIMS] + if arg_dims is not None: + for arg_dim in arg_dims: + if not arg_dim.isdigit() and arg_dim not in arg_names and arg_dim != ":": + global_ints_to_insert.append((arg_dim, "integer", "in", None)) + arg_names.add(arg_dim) + + db[active_sub] = global_ints_to_insert + ordered_decls active_sub = None result_name = None arg_decls = [] @@ -921,17 +983,38 @@ def gen_arg_f90_decl(argtype, intent, dims, names): 'integer(kind=c_int) , intent(inout) :: barg' >>> gen_arg_f90_decl("integer", "out", None, ["barg"]) 'integer(kind=c_int) , intent(out) :: barg' + >>> gen_arg_f90_decl('type::element_t', 'inout', (':',), ["foo"]) + 'type(c_ptr) , intent(inout), dimension(:) :: foo' """ - expect(argtype in C_TYPE_MAP, "Unrecognized argtype for C_TYPE_MAP: {}".format(argtype)) - c_type = C_TYPE_MAP[argtype] value = ", value" if dims is None and intent == "in" else "" - intent_s = ", intent({})".format(intent) - dimension_s = ", dimension({})".format(", ".join(dims)) if dims is not None else "" + intent_s = f", intent({intent})" + dimension_s = f", dimension({', '.join(dims)})" if dims is not None else "" names_s = ", ".join(names) - return "{argtype}(kind={c_type}) {value}{intent}{dimension} :: {names}".\ - format(argtype=argtype, c_type=c_type, value=value, intent=intent_s, dimension=dimension_s, names=names_s) + + if is_custom_type(argtype): + return f"type(c_ptr) {intent_s}{dimension_s} :: {names_s}" + else: + expect(argtype in C_TYPE_MAP, f"Unrecognized argtype for C_TYPE_MAP: {argtype}") + c_type = C_TYPE_MAP[argtype] + return f"{argtype}(kind={c_type}) {value}{intent_s}{dimension_s} :: {names_s}" + +############################################################################### +def is_custom_type(arg_type): +############################################################################### + return arg_type.startswith("type::") CXX_TYPE_MAP = {"real" : "Real", "integer" : "Int", "logical" : "bool"} +############################################################################### +def get_cxx_scalar_type(arg_type): +############################################################################### + if is_custom_type(arg_type): + arg_cxx_type = arg_type.split("::")[-1] + else: + expect(arg_type in CXX_TYPE_MAP, f"Unrecognized argtype for CXX_TYPE_MAP: {arg_type}") + arg_cxx_type = CXX_TYPE_MAP[arg_type] + + return arg_cxx_type + ############################################################################### def get_cxx_type(arg_datum): ############################################################################### @@ -952,12 +1035,13 @@ def get_cxx_type(arg_datum): 'Real*' >>> get_cxx_type(("foo", "integer", "inout", None)) 'Int*' + >>> get_cxx_type(('elem', 'type::element_t', 'inout', (':',))) + 'element_t*' """ is_ptr = arg_datum[ARG_DIMS] is not None or arg_datum[ARG_INTENT] != "in" arg_type = arg_datum[ARG_TYPE] - expect(arg_type in CXX_TYPE_MAP, "Unrecognized argtype for CXX_TYPE_MAP: {}".format(arg_type)) - arg_cxx_type = CXX_TYPE_MAP[arg_type] - return "{}{}".format(arg_cxx_type, "*" if is_ptr else "") + arg_cxx_type = get_cxx_scalar_type(arg_type) + return f"{arg_cxx_type}{'*' if is_ptr else ''}" KOKKOS_TYPE_MAP = {"real" : "Spack", "integer" : "Int", "logical" : "bool"} ############################################################################### @@ -983,14 +1067,22 @@ def get_kokkos_type(arg_datum): 'Spack&' >>> get_kokkos_type(("foo", "integer", "inout", None)) 'Int&' + >>> get_kokkos_type(('elem', 'type::element_t', 'inout', (':',))) + 'const uview_1d&' """ is_const = arg_datum[ARG_INTENT] == "in" is_view = arg_datum[ARG_DIMS] is not None - base_type = "{}{}".format("const " if is_const else "", KOKKOS_TYPE_MAP[arg_datum[ARG_TYPE]]) + arg_type = arg_datum[ARG_TYPE] + if is_custom_type(arg_type): + kokkos_type = arg_type.split("::")[-1] + else: + kokkos_type = KOKKOS_TYPE_MAP[arg_type] + + base_type = f"{'const ' if is_const else ''}{kokkos_type}" # We assume 1d even if the f90 array is 2d since we assume c++ will spawn a kernel # over one of the dimensions - return "const uview_1d<{}>&".format(base_type) if is_view else "{}&".format(base_type) + return f"const uview_1d<{base_type}>&" if is_view else f"{base_type}&" ############################################################################### def gen_arg_cxx_decls(arg_data, kokkos=False): @@ -1006,7 +1098,7 @@ def gen_arg_cxx_decls(arg_data, kokkos=False): arg_names = [item[ARG_NAME] for item in arg_data] get_type = get_kokkos_type if kokkos else get_cxx_type arg_types = [get_type(item) for item in arg_data] - arg_sig_list = ["{} {}".format(arg_type, arg_name) for arg_name, arg_type in zip(arg_names, arg_types)] + arg_sig_list = [f"{arg_type} {arg_name}" for arg_name, arg_type in zip(arg_names, arg_types)] return arg_sig_list ############################################################################### @@ -1049,7 +1141,7 @@ def split_by_intent(arg_data): elif intent == "out": outputs.append(name) else: - expect(False, "Unhandled intent: {}".format(intent)) + expect(False, f"Unhandled intent: {intent}") return inputs, inouts, outputs @@ -1070,8 +1162,10 @@ def split_by_type(arg_data): ints.append(name) elif argtype == "logical": logicals.append(name) + elif is_custom_type(argtype): + pass else: - expect(False, "Unhandled argtype: {}".format(argtype)) + expect(False, f"Unhandled argtype: {argtype}") return reals, ints, logicals @@ -1088,7 +1182,7 @@ def gen_cxx_data_args(physics, arg_data): args_needs_ptr = [item[ARG_DIMS] is None and item[ARG_INTENT] != "in" for item in arg_data] arg_names = [item[ARG_NAME] for item in arg_data] arg_dim_call = [item[ARG_NAME] in all_dims for item in arg_data] - args = ["{}d.{}".format("&" if need_ptr else "", arg_name) + args = [f"{'&' if need_ptr else ''}d.{arg_name}" for arg_name, need_ptr, dim_call in zip(arg_names, args_needs_ptr, arg_dim_call)] return args @@ -1163,12 +1257,12 @@ def gen_struct_members(arg_data): result = [] for intent, comment in intent_order: if intent in metadata: - result.append("// {}".format(comment)) + result.append(f"// {comment}") type_map = metadata[intent] for type_info, names in type_map.items(): type_name, is_ptr = type_info - decl_str = CXX_TYPE_MAP[type_name] - decl_str += " {};".format(", ".join(["{}{}".format("*" if is_ptr else "", name) for name in names])) + decl_str = get_cxx_scalar_type(type_name) + decl_str += f" {', '.join(['{}{}'.format('*' if is_ptr else '', name) for name in names])};" result.append(decl_str) result.append("") @@ -1176,7 +1270,7 @@ def gen_struct_members(arg_data): return result ############################################################################### -def group_data(arg_data, filter_out_intent=None): +def group_data(arg_data, filter_out_intent=None, filter_scalar_custom_types=False): ############################################################################### r""" Given data, return ([fst_dims], [snd_dims], [trd_dims], [all-dims], [scalars], {dims->[real_data]}, {dims->[int_data]}, {dims->[bool_data]}) @@ -1209,7 +1303,7 @@ def group_data(arg_data, filter_out_intent=None): for name, argtype, _, dims in arg_data: if dims is not None: expect(len(dims) >= 1 and len(dims) <= 3, - "Only 1d-3d data is supported, {} has too many dims: {}".format(name, len(dims))) + f"Only 1d-3d data is supported, {name} has too many dims: {len(dims)}") if dims[0] not in fst_dims: fst_dims.append(dims[0]) @@ -1226,10 +1320,11 @@ def group_data(arg_data, filter_out_intent=None): for name, argtype, intent, dims in arg_data: if filter_out_intent is None or intent != filter_out_intent: if dims is None: - if name not in all_dims: - scalars.append( (name, CXX_TYPE_MAP[argtype])) - else: - expect(argtype == "integer", "Expected dimension {} to be of type integer".format(name)) + if not (is_custom_type(argtype) and filter_scalar_custom_types): + if name not in all_dims: + scalars.append( (name, get_cxx_scalar_type(argtype))) + else: + expect(argtype == "integer", f"Expected dimension {name} to be of type integer") elif argtype == "integer": int_data.setdefault(dims, []).append(name) @@ -1237,7 +1332,7 @@ def group_data(arg_data, filter_out_intent=None): elif argtype == "real": real_data.setdefault(dims, []).append(name) - else: + elif argtype == "logical": bool_data.setdefault(dims, []).append(name) return fst_dims, snd_dims, trd_dims, all_dims, scalars, real_data, int_data, bool_data @@ -1254,7 +1349,7 @@ def gen_struct_api(physics, struct_name, arg_data): PTD_STD_DEF(DataSubName, 8, shcol, nlev, nlevi, ntracers, gag, bab1, bab2, val); """ - _, _, _, all_dims, scalars, real_data, int_data, bool_data = group_data(arg_data) + _, _, _, all_dims, scalars, real_data, int_data, bool_data = group_data(arg_data, filter_scalar_custom_types=True) result = [] dim_args = [(item, "Int") for item in all_dims if item is not None] @@ -1269,17 +1364,17 @@ def gen_struct_api(physics, struct_name, arg_data): bool_vec = [] for data, data_vec in zip([real_data, int_data, bool_data], [real_vec, int_vec, bool_vec]): for dims, items in data.items(): - dim_cxx_vec.append("{{ {} }}".format(", ".join(["{}_".format(item) for item in dims]))) - data_vec.append("{{ {} }}".format(", ".join(["&{}".format(item) for item in items]))) + dim_cxx_vec.append(f"{{ {', '.join(['{}_'.format(item) for item in dims])} }}") + data_vec.append(f"{{ {', '.join(['&{}'.format(item) for item in items])} }}") - parent_call = " PhysicsTestData({{{}}}, {{{}}}".format(", ".join(dim_cxx_vec), ", ".join(real_vec)) + parent_call = f" PhysicsTestData({{{', '.join(dim_cxx_vec)}}}, {{{', '.join(real_vec)}}}" if int_vec or bool_vec: - parent_call += ", {{{}}}".format(", ".join(int_vec)) + parent_call += f", {{{', '.join(int_vec)}}}" if bool_vec: - parent_call += ", {{{}}}".format(", ".join(bool_vec)) + parent_call += f", {{{', '.join(bool_vec)}}}" parent_call += ")" - parent_call += ", {}".format(", ".join(["{0}({0}_)".format(name) for name, _ in cons_args])) + parent_call += f", {', '.join(['{0}({0}_)'.format(name) for name, _ in cons_args])}" parent_call += " {}" result.append(parent_call) @@ -1332,8 +1427,7 @@ def check_existing_piece(lines, begin_regex, end_regex): if begin_match: expect(begin_idx is None, - "Found multiple begin matches for pattern '{}' before end pattern '{}' was found".\ - format(begin_regex.pattern, end_regex.pattern)) + f"Found multiple begin matches for pattern '{begin_regex.pattern}' before end pattern '{end_regex.pattern}' was found") begin_idx = idx @@ -1343,8 +1437,7 @@ def check_existing_piece(lines, begin_regex, end_regex): if begin_idx is not None: expect(end_idx is not None, - "Found no ending match for begin pattern '{}' starting on line {} and searching end pattern '{}'".\ - format(begin_regex.pattern, begin_idx, end_regex.pattern)) + "Found no ending match for begin pattern '{begin_regex.pattern}' starting on line {begin_idx} and searching end pattern '{end_regex.pattern}'") return None if begin_idx is None else (begin_idx, end_idx+1) @@ -1372,11 +1465,11 @@ def __init__(self, expect(target_repo is not None, "Must either run from a valid repo or provide a --target-repo") normalized_source_repo = get_git_toplevel_dir(repo=source_repo) - expect(normalized_source_repo is not None, "source repo {} is not a valid repo".format(source_repo)) + expect(normalized_source_repo is not None, f"source repo {source_repo} is not a valid repo") source_repo = normalized_source_repo normalized_target_repo = get_git_toplevel_dir(repo=target_repo) - expect(normalized_target_repo is not None, "target repo {} is not a valid repo".format(target_repo)) + expect(normalized_target_repo is not None, f"target repo {target_repo} is not a valid repo") target_repo = normalized_target_repo # configuration @@ -1385,8 +1478,8 @@ def __init__(self, self._physics = physics self._overwrite = overwrite self._kernel = kernel - self._source_repo = pathlib.Path(source_repo).resolve() - self._target_repo = pathlib.Path(target_repo).resolve() + self._source_repo = Path(source_repo).resolve() + self._target_repo = Path(target_repo).resolve() self._dry_run = dry_run self._verbose = verbose @@ -1403,28 +1496,30 @@ def __init__(self, ########################################################################### def _get_db(self, phys): ########################################################################### - if phys in self._db: - return self._db[phys] - else: - origin_file = self._source_repo / get_physics_data(phys, ORIGIN_FILE) - expect(origin_file.exists(), "Missing origin file for physics {}: {}".format(phys, origin_file)) - db = parse_origin(origin_file.open(encoding="utf-8").read(), self._subs) - self._db[phys] = db - if self._verbose: - print("For physics {}, found:") - for sub in self._subs: - if sub in db: - print(" For subroutine {}, found args:") - for name, argtype, intent, dims in db[sub]: - print(" name:{} type:{} intent:{} dims:({})".\ - format(name, argtype, intent, ",".join(dims) if dims else "scalar")) - return db + if phys not in self._db: + origin_files = get_physics_data(phys, ORIGIN_FILES) + self._db[phys] = {} + for origin_file in origin_files: + origin_file = self._source_repo / origin_file + expect(origin_file.exists(), f"Missing origin file for physics {phys}: {origin_file}") + db = parse_origin(origin_file.open(encoding="utf-8").read(), self._subs) + self._db[phys].update(db) + if self._verbose: + print("For physics {}, found:") + for sub in self._subs: + if sub in db: + print(" For subroutine {}, found args:") + for name, argtype, intent, dims in db[sub]: + print(" name:{} type:{} intent:{} dims:({})".\ + format(name, argtype, intent, ",".join(dims) if dims else "scalar")) + + return self._db[phys] ########################################################################### def _get_arg_data(self, phys, sub): ########################################################################### phys_db = self._get_db(phys) - expect(sub in phys_db, "No data for subroutine {} in physics {}".format(sub, phys)) + expect(sub in phys_db, f"No data for subroutine {sub} in physics {phys}") return phys_db[sub] ########################################################################### @@ -1435,7 +1530,7 @@ def dry_run(self): ############################################################################### def get_path_for_piece_file(self, physics, sub, piece): ############################################################################### - root_dir = pathlib.Path(get_physics_data(physics, CXX_ROOT)) + root_dir = Path(get_physics_data(physics, CXX_ROOT)) filepath = self._target_repo / root_dir / get_piece_data(physics, sub, piece, FILEPATH, self) return filepath @@ -1529,7 +1624,7 @@ def gen_cxx_c2f_bind_decl(self, phys, sub, force_arg_data=None): """ arg_data = force_arg_data if force_arg_data else self._get_arg_data(phys, sub) arg_decls = gen_arg_cxx_decls(arg_data) - result = "void {sub}_c({arg_sig});\n".format(sub=sub, arg_sig=", ".join(arg_decls)) + result = f"void {sub}_c({', '.join(arg_decls)});\n" return result ########################################################################### @@ -1541,7 +1636,7 @@ def gen_cxx_c2f_glue_decl(self, phys, sub, force_arg_data=None): void fake_sub(FakeSubData& d); """ struct_name = get_data_struct_name(sub) - result = "void {sub}({struct_name}& d);".format(sub=sub, struct_name=struct_name) + result = f"void {sub}({struct_name}& d);" return result ########################################################################### @@ -1567,16 +1662,15 @@ def gen_cxx_c2f_glue_impl(self, phys, sub, force_arg_data=None): transpose_code_2 = "\n d.transpose();" if need_transpose else "" data_struct = get_data_struct_name(sub) init_code = get_physics_data(phys, INIT_CODE) - init_code = init_code.replace("REPLACE_ME", "d.nlev") result = \ -"""void {sub}({data_struct}& d) +f"""void {sub}({data_struct}& d) {{ {init_code}{transpose_code_1} {sub}_c({arg_data_args});{transpose_code_2} }} -""".format(sub=sub, data_struct=data_struct, init_code=init_code, transpose_code_1=transpose_code_1, transpose_code_2=transpose_code_2, arg_data_args=arg_data_args) +""" return result ########################################################################### @@ -1617,11 +1711,11 @@ def gen_cxx_c2f_data(self, phys, sub, force_arg_data=None): api = "\n " + "\n ".join(gen_struct_api(phys, struct_name, arg_data) if any_arrays else "") result = \ -"""struct {struct_name}{inheritance} {{ +f"""struct {struct_name}{inheritance} {{ {struct_members}{api} }}; -""".format(struct_name=struct_name, inheritance=inheritance, struct_members=struct_members, api=api) +""" return result ########################################################################### @@ -1635,7 +1729,7 @@ def gen_cxx_f2c_bind_decl(self, phys, sub, force_arg_data=None): arg_data = force_arg_data if force_arg_data else self._get_arg_data(phys, sub) arg_decls = gen_arg_cxx_decls(arg_data) - return "void {sub}_f({arg_sig});".format(sub=sub, arg_sig=", ".join(arg_decls)) + return f"void {sub}_f({', '.join(arg_decls)});" ########################################################################### def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): @@ -1748,21 +1842,21 @@ def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): # make necessary view types for output_group, prefix_char, typename in zip([oreals, oints, obools], prefix_list, type_list): if output_group: - impl += " using {}view_1d = typename PF::view_1d<{}>;\n".format(prefix_char, typename) + impl += f" using {prefix_char}view_1d = typename PF::view_1d<{typename}>;\n" impl += "\n" # make output views for host and device for output_group, prefix_char in zip([oreals, oints, obools], prefix_list): if output_group: - impl += ' {0}view_1d {0}t_d("{0}t_d", {1});\n'.format(prefix_char, len(output_group)) - impl += " const auto {0}t_h = Kokkos::create_mirror_view({0}t_d);\n".format(prefix_char) + impl += f' {prefix_char}view_1d {prefix_char}t_d("{prefix_char}t_d", {len(output_group)});\n' + impl += f" const auto {prefix_char}t_h = Kokkos::create_mirror_view({prefix_char}t_d);\n" impl += "\n" # inout data must be derefenced before the kernel for io_group, typename in zip([ioreals, ioints, iobools], type_list): if io_group: - impl += " {} {};\n".format(typename, ", ".join(["local_{0}(*{0})".format(item) for item in io_group])) + impl += f" {typename} {', '.join(['local_{0}(*{0})'.format(item) for item in io_group])};\n" # start a kernel impl += " Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) {\n" @@ -1772,17 +1866,17 @@ def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): # not be packed (like dt) for output_group, typename in zip([list(ireals) + list(ooreals), oints, obools], ktype_list): if output_group: - impl += " {} ".format(typename) + impl += f" {typename} " temp_cons = [] for item in output_group: if item in inouts: - temp_cons.append("{0}_(local_{0})".format(item)) + temp_cons.append(f"{item}_(local_{item})") elif item in outputs: - temp_cons.append("{0}_()".format(item)) + temp_cons.append(f"{item}_()") else: - temp_cons.append("{0}_({0})".format(item)) + temp_cons.append(f"{item}_({item})") - impl += "{};\n".format(", ".join(temp_cons)) + impl += f"{', '.join(temp_cons)};\n" # Make cxx call kernel_arg_names = [] @@ -1792,15 +1886,15 @@ def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): else: kernel_arg_names.append(arg_name + "_") - impl += " PF::{}({});\n".format(sub, ", ".join(kernel_arg_names)) + impl += f" PF::{sub}({', '.join(kernel_arg_names)});\n" # Load output data into views for output_group, prefix_char in zip([oreals, oints, obools], prefix_list): for idx, item in enumerate(output_group): if output_group == oreals: - impl += " {}t_d({}) = {}_[0];\n".format(prefix_char, idx, item) + impl += f" {prefix_char}t_d({idx}) = {item}_[0];\n" else: - impl += " {}t_d({}) = {}_;\n".format(prefix_char, idx, item) + impl += f" {prefix_char}t_d({idx}) = {item}_;\n" # finish kernel impl += " });\n" @@ -1808,21 +1902,21 @@ def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): # copy outputs back to host for output_group, prefix_char in zip([oreals, oints, obools], prefix_list): if output_group: - impl += " Kokkos::deep_copy({0}t_h, {0}t_d);\n".format(prefix_char) + impl += f" Kokkos::deep_copy({prefix_char}t_h, {prefix_char}t_d);\n" # copy from views into pointer args for output_group, prefix_char in zip([oreals, oints, obools], prefix_list): for idx, item in enumerate(output_group): - impl += " *{} = {}t_h({});\n".format(item, prefix_char, idx) + impl += f" *{item} = {prefix_char}t_h({idx});\n" impl += "#endif\n" result = \ -"""{decl} +f"""{decl} {{ {impl} }} -""".format(decl=decl, impl=impl) +""" return result ########################################################################### @@ -1837,7 +1931,7 @@ def gen_cxx_func_decl(self, phys, sub, force_arg_data=None): arg_data = force_arg_data if force_arg_data else self._get_arg_data(phys, sub) arg_decls = gen_arg_cxx_decls(arg_data, kokkos=True) - return " KOKKOS_FUNCTION\n static void {sub}({arg_sig});".format(sub=sub, arg_sig=", ".join(arg_decls)) + return f" KOKKOS_FUNCTION\n static void {sub}({', '.join(arg_decls)});" ########################################################################### def gen_cxx_incl_impl(self, phys, sub, force_arg_data=None): @@ -1845,10 +1939,10 @@ def gen_cxx_incl_impl(self, phys, sub, force_arg_data=None): """ >>> gb = GenBoiler([]) >>> print(gb.gen_cxx_incl_impl("shoc", "fake_sub", force_arg_data=UT_ARG_DATA)) - # include "shoc_fake_sub_impl.hpp" + # include "impl/shoc_fake_sub_impl.hpp" """ impl_path = get_piece_data(phys, sub, "cxx_func_impl", FILEPATH, self) - return '# include "{}"'.format(impl_path) + return f'# include "{impl_path}"' ########################################################################### def gen_cxx_func_impl(self, phys, sub, force_arg_data=None): @@ -1868,11 +1962,11 @@ def gen_cxx_func_impl(self, phys, sub, force_arg_data=None): # I don't think any intelligent guess at an impl is possible here result = \ -"""{decl} +f"""{decl} {{ // TODO // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -}}""".format(decl=decl) +}}""" return result ########################################################################### @@ -1884,7 +1978,7 @@ def gen_cxx_bfb_unit_decl(self, phys, sub, force_arg_data=None): struct TestFakeSub; """ test_struct = get_data_test_struct_name(sub) - return " struct {};".format(test_struct) + return f" struct {test_struct};" ########################################################################### def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): @@ -1894,6 +1988,8 @@ def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): >>> print(gb.gen_cxx_bfb_unit_impl("shoc", "fake_sub", force_arg_data=UT_ARG_DATA)) static void run_bfb() { + auto engine = setup_random_test(); + FakeSubData f90_data[] = { // TODO }; @@ -1903,7 +1999,7 @@ def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): // Generate random input data // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data for (auto& d : f90_data) { - d.randomize(); + d.randomize(engine); } // Create copies of data for use by cxx. Needs to happen before fortran calls so that @@ -1942,12 +2038,15 @@ def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): REQUIRE(d_f90.total(d_f90.baz) == d_cxx.total(d_cxx.ball2)); REQUIRE(d_f90.ball2[k] == d_cxx.ball2[k]); } + } } } // run_bfb >>> print(gb.gen_cxx_bfb_unit_impl("shoc", "fake_sub", force_arg_data=UT_ARG_DATA_ALL_SCALAR)) static void run_bfb() { + auto engine = setup_random_test(); + FakeSubData f90_data[max_pack_size] = { // TODO }; @@ -1957,7 +2056,7 @@ def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): // Generate random input data // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data for (auto& d : f90_data) { - d.randomize(); + d.randomize(engine); } // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that @@ -2036,13 +2135,13 @@ def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): // Generate random input data // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data for (auto& d : f90_data) { - d.randomize(); + d.randomize(engine); }""" _, _, _, _, scalars, real_data, int_data, bool_data = group_data(arg_data, filter_out_intent="in") check_scalars, check_arrays = "", "" for scalar in scalars: - check_scalars += " REQUIRE(d_f90.{name} == d_cxx.{name});\n".format(name=scalar[0]) + check_scalars += f" REQUIRE(d_f90.{scalar[0]} == d_cxx.{scalar[0]});\n" if has_array: c2f_transpose_code = "" if not need_transpose else \ @@ -2061,17 +2160,19 @@ def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): all_data[k] = v for _, data in all_data.items(): - check_arrays += " for (Int k = 0; k < d_f90.total(d_f90.{}); ++k) {{\n".format(data[0]) + check_arrays += f" for (Int k = 0; k < d_f90.total(d_f90.{data[0]}); ++k) {{\n" for datum in data: - check_arrays += " REQUIRE(d_f90.total(d_f90.{orig}) == d_cxx.total(d_cxx.{name}));\n".format(orig=data[0], name=datum) - check_arrays += " REQUIRE(d_f90.{name}[k] == d_cxx.{name}[k]);\n".format(name=datum) + check_arrays += f" REQUIRE(d_f90.total(d_f90.{data[0]}) == d_cxx.total(d_cxx.{datum}));\n" + check_arrays += f" REQUIRE(d_f90.{datum}[k] == d_cxx.{datum}[k]);\n" - check_arrays += " }" + check_arrays += " }\n" if has_array: result = \ """ static void run_bfb() {{ + auto engine = setup_random_test(); + {data_struct} f90_data[] = {{ // TODO }}; @@ -2136,12 +2237,12 @@ def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): spack_output_init = "" if ooreals: spack_output_init = \ -"""// Init outputs - Spack {}; -""".format(", ".join(["{}(0)".format(ooreal) for ooreal in ooreals])) +f"""// Init outputs + Spack {', '.join(['{}(0)'.format(ooreal) for ooreal in ooreals])}; +""" scalars = group_data(arg_data)[4] - func_call = "Functions::{}({});".format(sub, ", ".join([(scalar if scalar in reals else "cxx_device(0).{}".format(scalar)) for scalar, _ in scalars])) + func_call = f"Functions::{sub}({', '.join([(scalar if scalar in reals else 'cxx_device(0).{}'.format(scalar)) for scalar, _ in scalars])});" spack_output_to_dview = "" if oreals: @@ -2155,6 +2256,8 @@ def gen_cxx_bfb_unit_impl(self, phys, sub, force_arg_data=None): result = \ """ static void run_bfb() {{ + auto engine = setup_random_test(); + {data_struct} f90_data[max_pack_size] = {{ // TODO }}; @@ -2211,7 +2314,7 @@ def gen_cxx_eti(self, phys, sub, force_arg_data=None): """ >>> gb = GenBoiler([]) >>> print(gb.gen_cxx_eti("shoc", "fake_sub", force_arg_data=UT_ARG_DATA)) - #include "shoc_fake_sub_impl.hpp" + #include "impl/shoc_fake_sub_impl.hpp" namespace scream { namespace shoc { @@ -2230,7 +2333,7 @@ def gen_cxx_eti(self, phys, sub, force_arg_data=None): include_file = get_piece_data(phys, sub, "cxx_func_impl", FILEPATH, self) result = \ -"""#include "{include_file}" +f"""#include "{include_file}" namespace scream {{ namespace {phys} {{ @@ -2244,7 +2347,7 @@ def gen_cxx_eti(self, phys, sub, force_arg_data=None): }} // namespace {phys} }} // namespace scream -""".format(sub=sub, include_file=include_file, phys=phys) +""" return result @@ -2254,10 +2357,10 @@ def gen_cmake_impl_eti(self, phys, sub, force_arg_data=None): """ >>> gb = GenBoiler([]) >>> print(gb.gen_cmake_impl_eti("shoc", "fake_sub", force_arg_data=UT_ARG_DATA)) - shoc_fake_sub.cpp + eti/shoc_fake_sub.cpp """ eti_src = get_piece_data(phys, sub, "cxx_eti", FILEPATH, self) - return " {}".format(eti_src) + return f" {eti_src}" ########################################################################### def gen_cmake_unit_test(self, phys, sub, force_arg_data=None): @@ -2267,8 +2370,8 @@ def gen_cmake_unit_test(self, phys, sub, force_arg_data=None): >>> print(gb.gen_cmake_unit_test("shoc", "fake_sub", force_arg_data=UT_ARG_DATA)) shoc_fake_sub_tests.cpp """ - test_src = os.path.basename(get_piece_data(phys, sub, "cxx_bfb_unit_impl", FILEPATH, self)) - return " {}".format(test_src) + test_src = Path(get_piece_data(phys, sub, "cxx_bfb_unit_impl", FILEPATH, self)).name + return f" {test_src}" # # Main methods @@ -2353,7 +2456,7 @@ def gen_piece(self, phys, sub, piece, force_arg_data=None, force_file_lines=None """ if force_arg_data is None: # don't want unit tests printing this print("===============================================================================") - print("Trying to generate piece {} for subroutine {} for physics {}\n".format(piece, sub, phys)) + print(f"Trying to generate piece {piece} for subroutine {sub} for physics {phys}\n") base_filepath, was_filegen, insert_regex, self_begin_regex, self_end_regex, _ \ = [item(phys, sub, self) for item in PIECES[piece]] @@ -2366,13 +2469,13 @@ def gen_piece(self, phys, sub, piece, force_arg_data=None, force_file_lines=None else: orig_lines = force_file_lines if force_file_lines else filepath.open(encoding="utf-8").read().splitlines() needs_rewrite = False - gen_lines = getattr(self, "gen_{}".format(piece))(phys, sub, force_arg_data=force_arg_data).splitlines() + gen_lines = getattr(self, f"gen_{piece}")(phys, sub, force_arg_data=force_arg_data).splitlines() # Check to see if piece already exists try: existing_piece_line_range = check_existing_piece(orig_lines, self_begin_regex, self_end_regex) except SystemExit as e: - expect(False, "Problem parsing file {} for existing piece {}: {}".format(filepath, piece, e)) + expect(False, f"Problem parsing file {filepath} for existing piece {piece}: {e}") if existing_piece_line_range is not None: # Replace existing @@ -2414,8 +2517,7 @@ def gen_boiler(self): try: self.gen_piece(phys, sub, piece) except SystemExit as e: - print("Warning: failed to generate subroutine {} piece {} for physics {}, error: {}".\ - format(sub, piece, phys, e)) + print(f"Warning: failed to generate subroutine {sub} piece {piece} for physics {phys}, error: {e}") all_success = False return all_success diff --git a/components/eamxx/scripts/jenkins/chrysalis_setup b/components/eamxx/scripts/jenkins/chrysalis_setup new file mode 100644 index 000000000000..e32f2a7083e4 --- /dev/null +++ b/components/eamxx/scripts/jenkins/chrysalis_setup @@ -0,0 +1,7 @@ +source /lcrc/soft/climate/e3sm-unified/load_latest_cime_env.sh + +source /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/init/sh + +export PROJECT=e3sm + +SCREAM_MACHINE=chrysalis diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index dfd5cbb178a6..fe293debe446 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -88,12 +88,17 @@ if [ $skip_testing -eq 0 ]; then fi fi + SA_FAILURES_DETAILS="" # Run scream stand-alone tests (SA) if [ $test_SA -eq 1 ]; then - ./scripts/gather-all-data "./scripts/test-all-scream ${TAS_ARGS}" -l -m $SCREAM_MACHINE + this_output=$(./scripts/gather-all-data "./scripts/test-all-scream ${TAS_ARGS}" -l -m $SCREAM_MACHINE) if [[ $? != 0 ]]; then fails=$fails+1; sa_fail=1 + if [[ $is_at_run == 1 ]]; then + errors=$(echo "$this_output" | grep -m1 -A 100000 'Build type ') + SA_FAILURES_DETAILS+="$errors" + fi fi # Add memcheck and coverage tests for nightlies on specific machines @@ -116,7 +121,7 @@ if [ $skip_testing -eq 0 ]; then fi if [[ "$SCREAM_MACHINE" == "weaver" ]]; then - ./scripts/gather-all-data "./scripts/test-all-scream -t cmc -t csr -t csi -t css ${TAS_ARGS}" -l -m $SCREAM_MACHINE + ./scripts/gather-all-data "./scripts/test-all-scream -t csm -t csr -t csi -t css ${TAS_ARGS}" -l -m $SCREAM_MACHINE if [[ $? != 0 ]]; then fails=$fails+1; memcheck_fail=1 @@ -190,10 +195,14 @@ if [ $skip_testing -eq 0 ]; then if [[ $test_v1 == 1 ]]; then # AT runs should be fast. => run only low resolution - ../../cime/scripts/create_test e3sm_scream_v1_lowres --compiler=gnu9 -c -b master --wait + this_output=$(../../cime/scripts/create_test e3sm_scream_v1_at --compiler=gnu9 -c -b master --wait) if [[ $? != 0 ]]; then fails=$fails+1; v1_fail=1 + if [[ $is_at_run == 1 ]]; then + errors=$(echo "$this_output" | grep -m1 -A 100000 'Waiting for tests to finish') + V1_FAILURES_DETAILS+="$errors" + fi fi else echo "SCREAM v1 tests were skipped, since the Github label 'AT: Skip v1 Testing' was found.\n" @@ -209,9 +218,11 @@ if [ $skip_testing -eq 0 ]; then echo "FAILS DETECTED:" if [[ $sa_fail == 1 ]]; then echo " SCREAM STANDALONE TESTING FAILED!" + echo "$SA_FAILURES_DETAILS" fi if [[ $v1_fail == 1 ]]; then echo " SCREAM V1 TESTING FAILED!" + echo "$V1_FAILURES_DETAILS" fi if [[ $v0_fail == 1 ]]; then echo " SCREAM V0 TESTING FAILED!" diff --git a/components/eamxx/scripts/jenkins/weaver_setup b/components/eamxx/scripts/jenkins/weaver_setup index daa5483de6ae..22f1e996b8b9 100644 --- a/components/eamxx/scripts/jenkins/weaver_setup +++ b/components/eamxx/scripts/jenkins/weaver_setup @@ -1,3 +1,5 @@ -module load python/3.7.3 +source /etc/profile.d/modules.sh + +module load python/3.10.8 SCREAM_MACHINE=weaver diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index d9566ad1a256..cd717cba6b97 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -21,24 +21,27 @@ ["mpicxx","mpifort","mpicc"], "salloc -N 1 srun -n1 --preserve-env", "/home/projects/e3sm/scream/pr-autotester/master-baselines/blake/"), - "weaver" : (["source /etc/profile.d/modules.sh", "module purge", "module load git/2.10.1 python/3.7.3 cmake/3.23.1 cuda/11.2.2/gcc/8.3.1 openmpi/4.1.1/gcc/8.3.1/cuda/11.2.2 netcdf-c/4.8.1/gcc/8.3.1/openmpi/4.1.1 netcdf-cxx/4.2/gcc/8.3.1/openmpi/4.1.1 netcdf-fortran/4.5.4/gcc/8.3.1/openmpi/4.1.1 parallel-netcdf/1.12.2/gcc/8.3.1/openmpi/4.1.1", + "weaver" : (["source /etc/profile.d/modules.sh", "module purge", "module load cmake/3.25.1 git/2.39.1 python/3.10.8 py-netcdf4/1.5.8 gcc/11.3.0 cuda/11.8.0 openmpi netcdf-c netcdf-fortran parallel-netcdf netlib-lapack", ], ["mpicxx","mpifort","mpicc"], - "bsub -I -q rhel8 -n 4", + "bsub -I -q rhel8 -n 4 -gpu num=4", "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["module purge", "module load sems-archive-env acme-env sems-archive-gcc/9.2.0 sems-archive-cmake/3.19.1 sems-archive-git/2.10.1 acme-openmpi/4.0.7 acme-netcdf/4.7.4/acme"], + "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 sems-archive-gcc/9.2.0 sems-archive-git/2.10.1 acme-openmpi/4.0.7 acme-netcdf/4.7.4/acme"], ["mpicxx","mpifort","mpicc"], "", "/sems-data-store/ACME/baselines/scream/master-baselines"), - "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/10.1.243 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'"], - ["mpicxx","mpifort","mpicc"], - "bsub -Ip -qpdebug", - ""), - "ruby-intel" : (["module --force purge", "module load StdEnv cmake/3.18.0 mkl/2019.0 intel/19.0.4 netcdf-fortran/4.4.4 netcdf/4.4.1.1 pnetcdf/1.9.0 mvapich2/2.3 python/3.8.2"], + "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'", + "export PATH=/usr/gdata/climdat/netcdf/bin:$PATH", + "export LD_LIBRARY_PATH=/usr/gdata/climdat/netcdf/lib:$LD_LIBRARY_PATH", + ], + ["mpicxx","mpifort","mpicc"], + "bsub -Ip -qpdebug", + ""), + "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), - "quartz-intel" : (["module --force purge", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic netcdf-c-parallel/4.9.0 netcdf-fortran-parallel/4.6.0 mvapich2/2.3.6 python/3.9.12"], + "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), @@ -54,14 +57,10 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q batch -W 0:30 -P cli115 -nnodes 1", "/gpfs/alpine/cli115/proj-shared/scream/master-baselines"), -"perlmutter" : (["module load PrgEnv-gnu gcc/10.3.0 cudatoolkit craype-accel-nvidia80 cray-libsci craype cray-mpich cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf cmake","module unload craype-accel-host perftools-base perftools darshan", "export NVCC_WRAPPER_DEFAULT_COMPILER=CC", "export NVCC_WRAPPER_DEFAULT_ARCH=sm_80"], + "pm-gpu" : (["module load PrgEnv-gnu gcc/10.3.0 cudatoolkit craype-accel-nvidia80 cray-libsci craype cray-mpich cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf cmake evp-patch","module unload craype-accel-host perftools-base perftools darshan", "export NVCC_WRAPPER_DEFAULT_COMPILER=CC", "export NVCC_WRAPPER_DEFAULT_ARCH=sm_80"], ["CC","ftn","cc"], "srun --time 00:30:00 --nodes=1 --constraint=gpu --exclusive -q regular --account e3sm_g", ""), - "cori-knl" : (["eval $(../../cime/CIME/Tools/get_case_env)", "export OMP_NUM_THREADS=68"], - ["CC","ftn","cc"], - "srun --time 02:00:00 --nodes=1 --constraint=knl,quad,cache --exclusive -q regular --account e3sm", - ""), "compy" : (["module purge", "module load cmake/3.19.6 gcc/8.1.0 mvapich2/2.3.1 python/3.7.3"], ["mpicxx","mpifort","mpicc"], "srun --time 02:00:00 --nodes=1 -p short --exclusive --account e3sm", diff --git a/components/eamxx/scripts/scripts-tests b/components/eamxx/scripts/scripts-tests index e80dbbb92b4f..b70a697a7d55 100755 --- a/components/eamxx/scripts/scripts-tests +++ b/components/eamxx/scripts/scripts-tests @@ -70,7 +70,7 @@ def test_cmake_cache_contents(test_obj, build_name, cache_var, expected_value): test_obj.assertTrue(cache_file.is_file(), "Missing cache file {}".format(cache_file)) # pylint: disable=no-member grep_output = run_cmd_assert_result(test_obj, "grep ^{} CMakeCache.txt".format(cache_var), from_dir=cache_file.parent) - value = grep_output.split("=")[-1] + value = grep_output.split("=", maxsplit=1)[-1] test_obj.assertEqual(expected_value.upper(), value.upper(), msg="For CMake cache variable {}, expected value '{}', got '{}'".format(cache_var, expected_value, value)) @@ -173,7 +173,7 @@ class TestBaseOuter: # Hides the TestBase class from test scanner def test_pylint(self): ensure_pylint() - run_cmd_assert_result(self, "python3 -m pylint --disable C --disable R {}".format(self._source_file), from_dir=TEST_DIR) + run_cmd_assert_result(self, "python3 -m pylint --disable C --disable R {}".format(self._source_file), from_dir=TEST_DIR, verbose=True) def test_gen_baseline(self): if self._generate: @@ -349,11 +349,12 @@ class TestTestAllScream(TestBaseOuter.TestBase): cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), self._machine, dry_run=False) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - builddir = "cuda_mem_check" if is_cuda_machine(self._machine) else "valgrind" + builddir = "compute_sanitizer_memcheck" if is_cuda_machine(self._machine) else "valgrind" test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, builddir, "SCREAM_TEST_SIZE", "SHORT") if is_cuda_machine(self._machine): - test_cmake_cache_contents(self, builddir, "EKAT_ENABLE_CUDA_MEMCHECK", "TRUE") + test_cmake_cache_contents(self, builddir, "EKAT_ENABLE_COMPUTE_SANITIZER", "TRUE") + test_cmake_cache_contents(self, builddir, "EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck") else: test_cmake_cache_contents(self, builddir, "EKAT_ENABLE_VALGRIND", "TRUE") else: diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index 94b642846265..dfade8beda0d 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -46,8 +46,13 @@ def __init__(self, longname, description, cmake_args, # A longer decription of the test self.description = description - # Cmake config args for this test + # Cmake config args for this test. Check that quoting is done with + # single quotes. self.cmake_args = cmake_args + for name, arg in self.cmake_args: + expect('"' not in arg, + f"In test definition for {longname}, found cmake args with double quotes {name}='{arg}'" + "Please use single quotes if quotes are needed.") # Does the test do baseline testing self.uses_baselines = uses_baselines @@ -172,21 +177,6 @@ def __init__(self, tas): if persistent_supp_file.exists(): self.cmake_args.append( ("EKAT_VALGRIND_SUPPRESSION_FILE", str(persistent_supp_file)) ) -############################################################################### -class CMC(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "cuda_mem_check", - "debug with cuda memcheck", - [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_CUDA_MEMCHECK", "True")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - ############################################################################### class CSM(TestProperty): ############################################################################### @@ -194,9 +184,11 @@ class CSM(TestProperty): def __init__(self, _): TestProperty.__init__( self, - "compute_santizer_memcheck", + "compute_sanitizer_memcheck", "debug with compute sanitizer memcheck", - [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True")], + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -209,11 +201,11 @@ class CSR(TestProperty): def __init__(self, _): TestProperty.__init__( self, - "compute_santizer_racecheck", + "compute_sanitizer_racecheck", "debug with compute sanitizer racecheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool racecheck'")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=racecheck --racecheck-detect-level=error'")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -226,11 +218,11 @@ class CSI(TestProperty): def __init__(self, _): TestProperty.__init__( self, - "compute_santizer_initcheck", + "compute_sanitizer_initcheck", "debug with compute sanitizer initcheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool initcheck'")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=initcheck")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -243,11 +235,11 @@ class CSS(TestProperty): def __init__(self, _): TestProperty.__init__( self, - "compute_santizer_synccheck", + "compute_sanitizer_synccheck", "debug with compute sanitizer synccheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool synccheck'")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=synccheck")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -363,12 +355,12 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._root_dir = Path(__file__).resolve().parent.parent else: self._root_dir = Path(self._root_dir).resolve() - expect(self._root_dir.is_dir() and self._root_dir.parts()[-2:] == ('scream', 'components'), + expect(self._root_dir.is_dir() and self._root_dir.parts()[-2:] == ("scream", "components"), f"Bad root-dir '{self._root_dir}', should be: $scream_repo/components/eamxx") # Make our test objects! Change mem to default mem-check test for current platform if "mem" in tests: - tests[tests.index("mem")] = "cmc" if self.on_cuda() else "valg" + tests[tests.index("mem")] = "csm" if self.on_cuda() else "valg" self._tests = test_factory(tests, self) if self._work_dir is not None: @@ -508,7 +500,7 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, else: if self._baseline_dir == "AUTO": - expect (self._baseline_ref is None or self._baseline_ref == 'origin/master', + expect (self._baseline_ref is None or self._baseline_ref == "origin/master", "Do not specify `-b XYZ` when using `--baseline-dir AUTO`. The AUTO baseline dir should be used for the master baselines only.\n" " `-b XYZ` needs to probably build baselines for ref XYZ. However, no baselines will be built if the dir already contains baselines.\n") # We treat the "AUTO" string as a request for automatic baseline dir. @@ -793,7 +785,7 @@ def create_ctest_resource_file(self, test, build_dir): data = {} # This is the only version numbering supported by ctest, so far - data['version'] = {"major":1,"minor":0} + data["version"] = {"major":1,"minor":0} # We add leading zeroes to ensure that ids will sort correctly # both alphabetically and numerically @@ -802,9 +794,9 @@ def create_ctest_resource_file(self, test, build_dir): devices.append({"id":f"{res_id:05d}"}) # Add resource groups - data['local'] = [{"devices":devices}] + data["local"] = [{"devices":devices}] - with (build_dir/"ctest_resource_file.json").open('w', encoding="utf-8") as outfile: + with (build_dir/"ctest_resource_file.json").open("w", encoding="utf-8") as outfile: json.dump(data,outfile,indent=2) return (end-start)+1 @@ -849,6 +841,7 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): # taskset range even though the ctest script is also running the tests if self._parallel: start, end = self.get_taskset_range(test) + result = result.replace("'", r"'\''") # handle nested quoting result = f"taskset -c {start}-{end} sh -c '{result}'" return result @@ -1037,7 +1030,7 @@ def get_last_ctest_file(self,test,phase): # of tie, $IDX as tiebreaker for file in files: file_no_path = file.name - tokens = re.split(r'_|-|\.',str(file_no_path)) + tokens = re.split(r"_|-|\.",str(file_no_path)) if latest is None: latest = file curr_tag = int(tokens[1]) diff --git a/components/eamxx/scripts/update-all-pip b/components/eamxx/scripts/update-all-pip old mode 100644 new mode 100755 diff --git a/components/eamxx/src/CMakeLists.txt b/components/eamxx/src/CMakeLists.txt index 59a5d6646443..9568d3cf85ce 100644 --- a/components/eamxx/src/CMakeLists.txt +++ b/components/eamxx/src/CMakeLists.txt @@ -4,6 +4,9 @@ add_subdirectory(dynamics) add_subdirectory(physics) add_subdirectory(diagnostics) add_subdirectory(control) +if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") + add_subdirectory(doubly-periodic) +endif() if (PROJECT_NAME STREQUAL "E3SM") add_subdirectory(mct_coupling) endif() diff --git a/components/eamxx/src/control/CMakeLists.txt b/components/eamxx/src/control/CMakeLists.txt index 281794f95ab9..19ce3b6dc318 100644 --- a/components/eamxx/src/control/CMakeLists.txt +++ b/components/eamxx/src/control/CMakeLists.txt @@ -1,14 +1,15 @@ set(SCREAM_CONTROL_SOURCES atmosphere_driver.cpp - fvphyshack.cpp atmosphere_surface_coupling_importer.cpp atmosphere_surface_coupling_exporter.cpp + intensive_observation_period.cpp surface_coupling_utils.cpp ) set(SCREAM_CONTROL_HEADERS atmosphere_driver.hpp atmosphere_surface_coupling.hpp + intensive_observation_period.hpp surface_coupling_utils.hpp ) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 0afb7a6328b8..96adc56127d4 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -2,6 +2,8 @@ #include "control/atmosphere_surface_coupling_importer.hpp" #include "control/atmosphere_surface_coupling_exporter.hpp" +#include "physics/share/physics_constants.hpp" + #include "share/atm_process/atmosphere_process_group.hpp" #include "share/atm_process/atmosphere_process_dag.hpp" #include "share/field/field_utils.hpp" @@ -22,9 +24,14 @@ // find blocks that eventually should be removed in favor of a design that // accounts for pg2. Some blocks may turn out to be unnecessary, and I simply // didn't realize I could do without the workaround. -#include "control/fvphyshack.hpp" +#include "share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp" + +#ifndef SCREAM_CIME_BUILD +#include +#endif #include +#include namespace scream { @@ -123,17 +130,6 @@ set_params(const ekat::ParameterList& atm_params) create_logger (); m_ad_status |= s_params_set; - -#ifdef SCREAM_CIME_BUILD - const auto pg_type = "PG2"; - fvphyshack = m_atm_params.sublist("grids_manager").get("physics_grid_type") == pg_type; - if (fvphyshack) { - // See the [rrtmgp active gases] note in dynamics/homme/atmosphere_dynamics_fv_phys.cpp. - fv_phys_rrtmgp_active_gases_init(m_atm_params); - } -#else - fvphyshack = false; -#endif } void AtmosphereDriver:: @@ -168,6 +164,49 @@ init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0) m_case_t0 = case_t0; } + + +void AtmosphereDriver:: +setup_intensive_observation_period () +{ + // At this point, must have comm, params, initialized timestamps, and grids created. + check_ad_status(s_comm_set | s_params_set | s_ts_inited | s_grids_created); + + // Check to make sure iop is not already initialized + EKAT_REQUIRE_MSG(not m_intensive_observation_period, "Error! setup_intensive_observation_period() is " + "called, but IOP already set up.\n"); + + // This function should only be called if we are enabling IOP + const bool enable_iop = + m_atm_params.sublist("driver_options").get("enable_intensive_observation_period", false); + EKAT_REQUIRE_MSG(enable_iop, "Error! setup_intensive_observation_period() is called, but " + "enable_intensive_observation_period=false " + "in driver_options parameters.\n"); + + // Params must include intensive_observation_period_options sublist. + const auto iop_sublist_exists = m_atm_params.isSublist("intensive_observation_period_options"); + EKAT_REQUIRE_MSG(iop_sublist_exists, + "Error! setup_intensive_observation_period() is called, but no intensive_observation_period_options " + "defined in parameters.\n"); + + const auto iop_params = m_atm_params.sublist("intensive_observation_period_options"); + const auto phys_grid = m_grids_manager->get_grid("Physics"); + const auto nlevs = phys_grid->get_num_vertical_levels(); + const auto hyam = phys_grid->get_geometry_data("hyam"); + const auto hybm = phys_grid->get_geometry_data("hybm"); + + m_intensive_observation_period = + std::make_shared(m_atm_comm, + iop_params, + m_run_t0, + nlevs, + hyam, + hybm); + + auto dx_short_f = phys_grid->get_geometry_data("dx_short"); + m_intensive_observation_period->set_grid_spacing(dx_short_f.get_view()()); +} + void AtmosphereDriver::create_atm_processes() { m_atm_logger->info("[EAMxx] create_atm_processes ..."); @@ -213,9 +252,12 @@ void AtmosphereDriver::create_grids() const auto& casename = ic_pl.get("restart_casename"); auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0); gm_params.set("ic_filename", filename); + m_atm_params.sublist("provenance").set("initial_conditions_file",filename); } else if (ic_pl.isParameter("Filename")) { // Initial run, if an IC file is present, pass it. - gm_params.set("ic_filename", ic_pl.get("Filename")); + auto filename = ic_pl.get("Filename"); + gm_params.set("ic_filename", filename); + m_atm_params.sublist("provenance").set("initial_conditions_file",filename); } m_atm_logger->debug(" [EAMxx] Creating grid manager '" + gm_type + "' ..."); @@ -229,6 +271,14 @@ void AtmosphereDriver::create_grids() m_atm_logger->debug(" [EAMxx] Grids created."); + // If TMS process is enabled, SHOC needs to know to request tms' surface drag coefficient + // as a required field during the set_grid() call below, but SHOC does not have knowledge + // of other processes. The driver needs propgate this information to SHOC. + if(m_atm_process_group->has_process("tms") && + m_atm_process_group->has_process("shoc")) { + setup_shoc_tms_links(); + } + // Set the grids in the processes. Do this by passing the grids manager. // Each process will grab what they need m_atm_process_group->set_grids(m_grids_manager); @@ -288,6 +338,10 @@ void AtmosphereDriver::setup_surface_coupling_processes () const std::shared_ptr importer = std::dynamic_pointer_cast(atm_proc); importer->setup_surface_coupling_data(*m_surface_coupling_import_data_manager); + + if (m_intensive_observation_period) { + importer->set_intensive_observation_period(m_intensive_observation_period); + } } if (atm_proc->type() == AtmosphereProcessType::SurfaceCouplingExporter) { exporter_found = true; @@ -314,7 +368,7 @@ void AtmosphereDriver::setup_surface_coupling_processes () const } } -void AtmosphereDriver::reset_accummulated_fields () +void AtmosphereDriver::reset_accumulated_fields () { constexpr Real zero = 0; for (auto fm_it : m_field_mgrs) { @@ -346,32 +400,20 @@ void AtmosphereDriver::setup_column_conservation_checks () // Get fields needed to run the mass and energy conservation checks. Require that // all fields exist. - const auto pseudo_density_ptr = phys_field_mgr->get_field_ptr("pseudo_density"); - const auto ps_ptr = phys_field_mgr->get_field_ptr("ps"); - const auto phis_ptr = phys_field_mgr->get_field_ptr("phis"); - const auto horiz_winds_ptr = phys_field_mgr->get_field_ptr("horiz_winds"); - const auto T_mid_ptr = phys_field_mgr->get_field_ptr("T_mid"); - const auto qv_ptr = phys_field_mgr->get_field_ptr("qv"); - const auto qc_ptr = phys_field_mgr->get_field_ptr("qc"); - const auto qr_ptr = phys_field_mgr->get_field_ptr("qr"); - const auto qi_ptr = phys_field_mgr->get_field_ptr("qi"); - const auto vapor_flux_ptr = phys_field_mgr->get_field_ptr("vapor_flux"); - const auto water_flux_ptr = phys_field_mgr->get_field_ptr("water_flux"); - const auto ice_flux_ptr = phys_field_mgr->get_field_ptr("ice_flux"); - const auto heat_flux_ptr = phys_field_mgr->get_field_ptr("heat_flux"); - EKAT_REQUIRE_MSG(pseudo_density_ptr != nullptr && - ps_ptr != nullptr && - phis_ptr != nullptr && - horiz_winds_ptr != nullptr && - T_mid_ptr != nullptr && - qv_ptr != nullptr && - qc_ptr != nullptr && - qr_ptr != nullptr && - qi_ptr != nullptr && - vapor_flux_ptr != nullptr && - water_flux_ptr != nullptr && - ice_flux_ptr != nullptr && - heat_flux_ptr != nullptr, + EKAT_REQUIRE_MSG ( + phys_field_mgr->has_field("pseudo_density") and + phys_field_mgr->has_field("ps") and + phys_field_mgr->has_field("phis") and + phys_field_mgr->has_field("horiz_winds") and + phys_field_mgr->has_field("T_mid") and + phys_field_mgr->has_field("qv") and + phys_field_mgr->has_field("qc") and + phys_field_mgr->has_field("qr") and + phys_field_mgr->has_field("qi") and + phys_field_mgr->has_field("vapor_flux") and + phys_field_mgr->has_field("water_flux") and + phys_field_mgr->has_field("ice_flux") and + phys_field_mgr->has_field("heat_flux"), "Error! enable_column_conservation_checks=true for some atm process, " "but not all fields needed for this check exist in the FieldManager.\n"); @@ -381,14 +423,28 @@ void AtmosphereDriver::setup_column_conservation_checks () const Real energy_error_tol = driver_options_pl.get("energy_column_conservation_error_tolerance", 1e-14); // Create energy checker + const auto pseudo_density = phys_field_mgr->get_field("pseudo_density"); + const auto ps = phys_field_mgr->get_field("ps"); + const auto phis = phys_field_mgr->get_field("phis"); + const auto horiz_winds = phys_field_mgr->get_field("horiz_winds"); + const auto T_mid = phys_field_mgr->get_field("T_mid"); + const auto qv = phys_field_mgr->get_field("qv"); + const auto qc = phys_field_mgr->get_field("qc"); + const auto qr = phys_field_mgr->get_field("qr"); + const auto qi = phys_field_mgr->get_field("qi"); + const auto vapor_flux = phys_field_mgr->get_field("vapor_flux"); + const auto water_flux = phys_field_mgr->get_field("water_flux"); + const auto ice_flux = phys_field_mgr->get_field("ice_flux"); + const auto heat_flux = phys_field_mgr->get_field("heat_flux"); + auto conservation_check = std::make_shared(phys_grid, mass_error_tol, energy_error_tol, - pseudo_density_ptr, ps_ptr, phis_ptr, - horiz_winds_ptr, T_mid_ptr, qv_ptr, - qc_ptr, qr_ptr, qi_ptr, - vapor_flux_ptr, water_flux_ptr, - ice_flux_ptr, heat_flux_ptr); + pseudo_density, ps, phis, + horiz_winds, T_mid, qv, + qc, qr, qi, + vapor_flux, water_flux, + ice_flux, heat_flux); //Get fail handling type from driver_option parameters. const std::string fail_handling_type_str = @@ -409,6 +465,37 @@ void AtmosphereDriver::setup_column_conservation_checks () m_atm_process_group->setup_column_conservation_checks(conservation_check, fail_handling_type); } +void AtmosphereDriver::setup_shoc_tms_links () +{ + EKAT_REQUIRE_MSG(m_atm_process_group->has_process("tms"), + "Error! Attempting to setup link between " + "SHOC and TMS, but TMS is not defined.\n"); + EKAT_REQUIRE_MSG(m_atm_process_group->has_process("shoc"), + "Error! Attempting to setup link between " + "SHOC and TMS, but SHOC is not defined.\n"); + + auto shoc_process = m_atm_process_group->get_process_nonconst("shoc"); + shoc_process->get_params().set("apply_tms", true); +} + +void AtmosphereDriver::add_additional_column_data_to_property_checks () { + // Get list of additional data fields from driver_options parameters. + // If no fields given, return. + using vos_t = std::vector; + auto additional_data_fields = m_atm_params.sublist("driver_options").get("property_check_data_fields", + {"NONE"}); + if (additional_data_fields == vos_t{"NONE"}) return; + + // Add requested fields to property checks + auto phys_field_mgr = m_field_mgrs[m_grids_manager->get_grid("Physics")->name()]; + for (auto fname : additional_data_fields) { + EKAT_REQUIRE_MSG(phys_field_mgr->has_field(fname), "Error! The field "+fname+" is requested for property check output " + "but does not exist in the physics field manager.\n"); + + m_atm_process_group->add_additional_data_fields_to_property_checks(phys_field_mgr->get_field(fname)); + } +} + void AtmosphereDriver::create_fields() { m_atm_logger->info("[EAMxx] create_fields ..."); @@ -468,20 +555,19 @@ void AtmosphereDriver::create_fields() // Loop over all fields in group src_name on grid src_grid. for (const auto& fname : rel_info->m_fields_names) { // Get field on src_grid - auto f = rel_fm->get_field_ptr(fname); + const auto& rel_fid = rel_fm->get_field_id(fname); // Build a FieldRequest for the same field on greq's grid, // and add it to the group of this request if (fvphyshack) { - const auto& sfid = f->get_header().get_identifier(); - auto dims = sfid.get_layout().dims(); + auto dims = rel_fid.get_layout().dims(); dims[0] = fm->get_grid()->get_num_local_dofs(); - FieldLayout fl(sfid.get_layout().tags(), dims); - FieldIdentifier fid(sfid.name(), fl, sfid.get_units(), req.grid); + FieldLayout fl(rel_fid.get_layout().tags(), dims); + FieldIdentifier fid(rel_fid.name(), fl, rel_fid.get_units(), req.grid); FieldRequest freq(fid,req.name,req.pack_size); fm->register_field(freq); } else { - const auto fid = r->create_tgt_fid(f->get_header().get_identifier()); + const auto fid = r->create_tgt_fid(rel_fid); FieldRequest freq(fid,req.name,req.pack_size); fm->register_field(freq); } @@ -594,9 +680,10 @@ void AtmosphereDriver::initialize_output_managers () { auto& io_params = m_atm_params.sublist("Scorpio"); - // IMPORTANT: create model restart OutputManager first! This OM will be able to - // retrieve the original simulation start date, which we later pass to the - // OM of all the requested outputs. + // IMPORTANT: create model restart OutputManager first! This OM will be in charge + // of creating rpointer.atm, while other OM's will simply append to it. + // If this assumption is not verified, we must always append to rpointer, which + // can make the rpointer file a bit confusing. // Check for model restart output ekat::ParameterList checkpoint_params; @@ -604,8 +691,8 @@ void AtmosphereDriver::initialize_output_managers () { checkpoint_params.set("Frequency",-1); if (io_params.isSublist("model_restart")) { auto restart_pl = io_params.sublist("model_restart"); - // Signal that this is not a normal output, but the model restart one m_output_managers.emplace_back(); + restart_pl.sublist("provenance") = m_atm_params.sublist("provenance"); auto& om = m_output_managers.back(); if (fvphyshack) { // Don't save CGLL fields from ICs to the restart file. @@ -646,6 +733,7 @@ void AtmosphereDriver::initialize_output_managers () { params.set("filename_prefix",m_casename+".scream.h"+std::to_string(om_tally)); om_tally++; } + params.sublist("provenance") = m_atm_params.sublist("provenance"); // Add a new output manager m_output_managers.emplace_back(); auto& om = m_output_managers.back(); @@ -660,6 +748,43 @@ void AtmosphereDriver::initialize_output_managers () { m_atm_logger->info("[EAMxx] initialize_output_managers ... done!"); } +void AtmosphereDriver:: +set_provenance_data (std::string caseid, + std::string hostname, + std::string username) +{ +#ifdef SCREAM_CIME_BUILD + // Check the inputs are valid + EKAT_REQUIRE_MSG (caseid!="", "Error! Invalid case id: " + caseid + "\n"); + EKAT_REQUIRE_MSG (hostname!="", "Error! Invalid hostname: " + hostname + "\n"); + EKAT_REQUIRE_MSG (username!="", "Error! Invalid username: " + username + "\n"); +#else + caseid = "EAMxx standalone"; + char* user = new char[32]; + char* host = new char[256]; + int err; + err = gethostname(host,255); + if (err==0) { + hostname = std::string(host); + } else { + hostname = "UNKNOWN"; + } + err = getlogin_r(user,31); + if (err==0) { + username = std::string(user); + } else { + username = "UNKNOWN"; + } + delete[] user; + delete[] host; +#endif + auto& provenance = m_atm_params.sublist("provenance"); + provenance.set("caseid",caseid); + provenance.set("hostname",hostname); + provenance.set("username",username); + provenance.set("version",std::string(EAMXX_GIT_VERSION)); +} + void AtmosphereDriver:: initialize_fields () { @@ -669,10 +794,8 @@ initialize_fields () start_timer("EAMxx::init"); start_timer("EAMxx::initialize_fields"); -#ifdef SCREAM_CIME_BUILD - // See the [rrtmgp active gases] note in dynamics/homme/atmosphere_dynamics_fv_phys.cpp. + // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp if (fvphyshack) fv_phys_rrtmgp_active_gases_set_restart(m_case_t0 < m_run_t0); -#endif // See if we need to print a DAG. We do this first, cause if any input // field is missing from the initial condition file, an error will be thrown. @@ -875,33 +998,31 @@ void AtmosphereDriver::set_initial_conditions () const auto& fname = fid.name(); const auto& grid_name = fid.get_grid_name(); - // First, check if the input file contains constant values for some of the fields if (ic_pl.isParameter(fname)) { - // The user provided a constant value for this field. Simply use that. + // This is the case that the user provided an initialization + // for this field in the parameter file. if (ic_pl.isType(fname) or ic_pl.isType>(fname)) { + // Initial condition is a constant initialize_constant_field(fid, ic_pl); - fields_inited[grid_name].push_back(fname); // Note: f is const, so we can't modify the tracking. So get the same field from the fm auto f_nonconst = m_field_mgrs.at(grid_name)->get_field(fid.name()); f_nonconst.get_header().get_tracking().update_time_stamp(m_current_ts); } else if (ic_pl.isType(fname)) { + // Initial condition is a string ic_fields_to_copy.push_back(fid); - fields_inited[grid_name].push_back(fname); } else { - EKAT_REQUIRE_MSG (false, "ERROR: invalid assignment for variable " + fname + ", only scalar double or string, or vector double arguments are allowed"); + EKAT_ERROR_MSG ("ERROR: invalid assignment for variable " + fname + ", only scalar " + "double or string, or vector double arguments are allowed"); } - } else if (not (fvphyshack and grid_name == "Physics PG2")) { - auto& this_grid_ic_fnames = ic_fields_names[grid_name]; + fields_inited[grid_name].push_back(fname); + } else if (fname == "phis" or fname == "sgh30") { + // Both phis and sgh30 need to be loaded from the topography file auto& this_grid_topo_file_fnames = topography_file_fields_names[grid_name]; auto& this_grid_topo_eamxx_fnames = topography_eamxx_fields_names[grid_name]; - auto c = f.get_header().get_children(); - if (fname == "phis") { - // Topography (phis) is a special case that should - // be loaded from the topography file, where the - // eamxx field "phis" corresponds to the name + // The eamxx field "phis" corresponds to the name // "PHIS_d" on the GLL and Point grids and "PHIS" // on the PG2 grid in the topography file. if (grid_name == "Physics PG2") { @@ -912,11 +1033,29 @@ void AtmosphereDriver::set_initial_conditions () } else { EKAT_ERROR_MSG ("Error! Requesting phis on an unknown grid: " + grid_name + ".\n"); } - this_grid_topo_eamxx_fnames.push_back("phis"); - } else if (c.size()==0) { + this_grid_topo_eamxx_fnames.push_back(fname); + fields_inited[grid_name].push_back(fname); + } else if (fname == "sgh30") { + // The eamxx field "sgh30" is called "SGH30" in the + // topography file and is only available on the PG2 grid. + EKAT_ASSERT_MSG(grid_name == "Physics PG2", + "Error! Requesting sgh30 field on " + grid_name + + " topo file only has sgh30 for Physics PG2.\n"); + topography_file_fields_names[grid_name].push_back("SGH30"); + topography_eamxx_fields_names[grid_name].push_back(fname); + fields_inited[grid_name].push_back(fname); + } + } else if (not (fvphyshack and grid_name == "Physics PG2")) { + // The IC file is written for the GLL grid, so we only load + // fields from there. Any other input fields on the PG2 grid + // will be properly computed in the dynamics interface. + auto& this_grid_ic_fnames = ic_fields_names[grid_name]; + auto c = f.get_header().get_children(); + if (c.size()==0) { // If this field is the parent of other subfields, we only read from file the subfields. if (not ekat::contains(this_grid_ic_fnames,fname)) { this_grid_ic_fnames.push_back(fname); + fields_inited[grid_name].push_back(fname); } } else if (fvphyshack and grid_name == "Physics GLL") { // [CGLL ICs in pg2] I tried doing something like this in @@ -930,10 +1069,10 @@ void AtmosphereDriver::set_initial_conditions () const auto& fname = fid.name(); if (ic_pl.isParameter(fname) and ic_pl.isType(fname)) { initialize_constant_field(fid, ic_pl); - fields_inited[grid_name].push_back(fname); } else { this_grid_ic_fnames.push_back(fname); } + fields_inited[grid_name].push_back(fname); } } } @@ -990,6 +1129,22 @@ void AtmosphereDriver::set_initial_conditions () } } + if (m_intensive_observation_period) { + // For runs with IOP, call to setup io grids and lat + // lon information needed for reading from file + for (const auto& it : m_field_mgrs) { + const auto& grid_name = it.first; + if (ic_fields_names[grid_name].size() > 0) { + const auto& file_name = grid_name == "Physics GLL" + ? + ic_pl.get("Filename") + : + ic_pl.get("topography_filename"); + m_intensive_observation_period->setup_io_info(file_name, it.second->get_grid()); + } + } + } + // If a filename is specified, use it to load inputs on all grids if (ic_pl.isParameter("Filename")) { // Now loop over all grids, and load from file the needed fields on each grid (if any). @@ -997,7 +1152,16 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" [EAMxx] IC filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - read_fields_from_file (ic_fields_names[grid_name],it.second->get_grid(),file_name,m_current_ts); + if (not m_intensive_observation_period) { + read_fields_from_file (ic_fields_names[grid_name],it.second->get_grid(),file_name,m_current_ts); + } else { + // For IOP enabled, we load from file and copy data from the closest + // lat/lon column to every other column + m_intensive_observation_period->read_fields_from_file_for_iop(file_name, + ic_fields_names[grid_name], + m_current_ts, + it.second); + } } } @@ -1063,20 +1227,31 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - // Topography files always use "ncol_d" for the GLL grid value of ncol. - // To ensure we read in the correct value, we must change the name for that dimension - auto io_grid = it.second->get_grid(); - if (grid_name=="Physics GLL") { - using namespace ShortFieldTagsNames; - auto grid = io_grid->clone(io_grid->name(),true); - grid->reset_field_tag_name(COL,"ncol_d"); - io_grid = grid; + if (not m_intensive_observation_period) { + // Topography files always use "ncol_d" for the GLL grid value of ncol. + // To ensure we read in the correct value, we must change the name for that dimension + auto io_grid = it.second->get_grid(); + if (grid_name=="Physics GLL") { + using namespace ShortFieldTagsNames; + auto grid = io_grid->clone(io_grid->name(),true); + grid->reset_field_tag_name(COL,"ncol_d"); + io_grid = grid; + } + read_fields_from_file (topography_file_fields_names[grid_name], + topography_eamxx_fields_names[grid_name], + io_grid,file_name,m_current_ts); + } else { + // For IOP enabled, we load from file and copy data from the closest + // lat/lon column to every other column + m_intensive_observation_period->read_fields_from_file_for_iop(file_name, + topography_file_fields_names[grid_name], + topography_eamxx_fields_names[grid_name], + m_current_ts, + it.second); } - - read_fields_from_file (topography_file_fields_names[grid_name], - topography_eamxx_fields_names[grid_name], - io_grid,file_name,m_current_ts); } + // Store in provenance list, for later usage in output file metadata + m_atm_params.sublist("provenance").set("topography_file",file_name); m_atm_logger->debug(" [EAMxx] Processing topography from file ... done!"); } else { // Ensure that, if no topography_filename is given, no @@ -1089,6 +1264,88 @@ void AtmosphereDriver::set_initial_conditions () "topography_filename or entry matching the field name " "was given in IC parameters.\n"); } + + m_atm_params.sublist("provenance").set("topography_file","NONE"); + } + + if (m_intensive_observation_period) { + // Load IOP data file data for initial time stamp + m_intensive_observation_period->read_iop_file_data(m_current_ts); + + // Now that ICs are processed, set appropriate fields using IOP file data. + // Since ICs are loaded on GLL grid, we set those fields only and dynamics + // will take care of the rest (for PG2 case). + if (m_field_mgrs.count("Physics GLL") > 0) { + const auto& fm = m_field_mgrs.at("Physics GLL"); + m_intensive_observation_period->set_fields_from_iop_data(fm); + } + } + + // Compute IC perturbations of GLL fields (if requested) + using vos = std::vector; + const auto perturbed_fields = ic_pl.get("perturbed_fields", {}); + const auto num_perturb_fields = perturbed_fields.size(); + if (num_perturb_fields > 0) { + m_atm_logger->info(" [EAMxx] Adding random perturbation to ICs ..."); + + EKAT_REQUIRE_MSG(m_field_mgrs.count("Physics GLL") > 0, + "Error! Random perturbation can only be applied to fields on " + "the GLL grid, but no Physics GLL FieldManager was defined.\n"); + const auto& fm = m_field_mgrs.at("Physics GLL"); + + // Setup RNG. There are two relevant params: generate_perturbation_random_seed and + // perturbation_random_seed. We have 3 cases: + // 1. Parameter generate_perturbation_random_seed is set true, assert perturbation_random_seed + // is not given and generate a random seed using std::rand() to get an integer random value. + // 2. Parameter perturbation_random_seed is given, use this value for the seed. + // 3. Parameter perturbation_random_seed is not given and generate_perturbation_random_seed is + // not given, use 0 as the random seed. + // Case 3 is considered the default (using seed=0). + int seed; + if (ic_pl.get("generate_perturbation_random_seed", false)) { + EKAT_REQUIRE_MSG(not ic_pl.isParameter("perturbation_random_seed"), + "Error! Param generate_perturbation_random_seed=true, and " + "a perturbation_random_seed is given. Only one of these can " + "be defined for a simulation.\n"); + std::srand(std::time(nullptr)); + seed = std::rand(); + } else { + seed = ic_pl.get("perturbation_random_seed", 0); + } + m_atm_logger->info(" For IC perturbation, random seed: "+std::to_string(seed)); + std::mt19937_64 engine(seed); + + // Get perturbation limit. Defines a range [1-perturbation_limit, 1+perturbation_limit] + // for which the perturbation value will be randomly generated from. Create a uniform + // distribution for this range. + const auto perturbation_limit = ic_pl.get("perturbation_limit", 0.001); + std::uniform_real_distribution pdf(1-perturbation_limit, 1+perturbation_limit); + + // Define a level mask using reference pressure and the perturbation_minimum_pressure parameter. + // This mask dictates which levels we apply a perturbation. + const auto gll_grid = m_grids_manager->get_grid("Physics GLL"); + const auto hyam_h = gll_grid->get_geometry_data("hyam").get_view(); + const auto hybm_h = gll_grid->get_geometry_data("hybm").get_view(); + constexpr auto ps0 = physics::Constants::P0; + const auto min_pressure = ic_pl.get("perturbation_minimum_pressure", 1050.0); + auto pressure_mask = [&] (const int ilev) { + const auto pref = (hyam_h(ilev)*ps0 + hybm_h(ilev)*ps0)/100; // Reference pressure ps0 is in Pa, convert to millibar + return pref > min_pressure; + }; + + // Loop through fields and apply perturbation. + for (size_t f=0; fget_grid()->name()], fname), + "Error! Attempting to apply perturbation to field not in initial_conditions.\n" + " - Field: "+fname+"\n" + " - Grid: "+fm->get_grid()->name()+"\n"); + + auto field = fm->get_field(fname); + perturb(field, engine, pdf, seed, pressure_mask, fm->get_grid()->get_dofs_gids()); + } + + m_atm_logger->info(" [EAMxx] Adding random perturbation to ICs ... done!"); } m_atm_logger->info(" [EAMxx] set_initial_conditions ... done!"); @@ -1125,6 +1382,7 @@ read_fields_from_file (const std::vector& field_names_nc, } AtmosphereInput ic_reader(file_name,grid,fields); + ic_reader.set_logger(m_atm_logger); ic_reader.read_variables(); ic_reader.finalize(); @@ -1163,6 +1421,7 @@ read_fields_from_file (const std::vector& field_names, } AtmosphereInput ic_reader(file_name,grid,fields); + ic_reader.set_logger(m_atm_logger); ic_reader.read_variables(); ic_reader.finalize(); @@ -1258,6 +1517,9 @@ void AtmosphereDriver::initialize_atm_procs () m_atm_process_group->add_postcondition_nan_checks(); } + // Add additional column data fields to pre/postcondition checks (if they exist) + add_additional_column_data_to_property_checks(); + if (fvphyshack) { // [CGLL ICs in pg2] See related notes in atmosphere_dynamics.cpp. const auto gn = "Physics GLL"; @@ -1282,6 +1544,7 @@ initialize (const ekat::Comm& atm_comm, { set_comm(atm_comm); set_params(params); + set_provenance_data (); init_scorpio (); @@ -1291,37 +1554,65 @@ initialize (const ekat::Comm& atm_comm, create_grids (); + const bool enable_iop = + m_atm_params.sublist("driver_options").get("enable_intensive_observation_period", false); + if (enable_iop) { + setup_intensive_observation_period (); + } + create_fields (); initialize_fields (); initialize_atm_procs (); + // Do this before init-ing the output managers, + // so the fields are valid if outputing at t=0 + reset_accumulated_fields(); + initialize_output_managers (); } void AtmosphereDriver::run (const int dt) { start_timer("EAMxx::run"); - // Zero out accumulated fields - reset_accummulated_fields(); - // Make sure the end of the time step is after the current start_time EKAT_REQUIRE_MSG (dt>0, "Error! Input time step must be positive.\n"); // Print current timestamp information m_atm_logger->log(ekat::logger::LogLevel::info, "Atmosphere step = " + std::to_string(m_current_ts.get_num_steps()) + "\n" + - " model time = " + m_current_ts.get_date_string() + " " + m_current_ts.get_time_string() + "\n"); + " model start-of-step time = " + m_current_ts.get_date_string() + " " + m_current_ts.get_time_string() + "\n"); + + // Reset accum fields to 0 + // Note: at the 1st timestep this is redundant, since we did it at init, + // to ensure t=0 INSTANT output was correct. However, it's not a + // very expensive operation, so it's not worth the effort of the + // nano-opt of removing the call for the 1st timestep. + reset_accumulated_fields(); // The class AtmosphereProcessGroup will take care of dispatching arguments to // the individual processes, which will be called in the correct order. m_atm_process_group->run(dt); + // Some accumulated fields need to be divided by dt at the end of the atm step + for (auto fm_it : m_field_mgrs) { + const auto& fm = fm_it.second; + if (not fm->has_group("DIVIDE_BY_DT")) { + continue; + } + + auto rescale_group = fm->get_field_group("DIVIDE_BY_DT"); + for (auto f_it : rescale_group.m_fields) { + f_it.second->scale(Real(1) / dt); + } + } + // Update current time stamps m_current_ts += dt; // Update output streams + m_atm_logger->debug("[EAMxx::run] running output managers..."); for (auto& out_mgr : m_output_managers) { out_mgr.run(m_current_ts); } @@ -1358,8 +1649,10 @@ void AtmosphereDriver::finalize ( /* inputs? */ ) { m_output_managers.clear(); // Finalize, and then destroy all atmosphere processes - m_atm_process_group->finalize( /* inputs ? */ ); - m_atm_process_group = nullptr; + if (m_atm_process_group.get()) { + m_atm_process_group->finalize( /* inputs ? */ ); + m_atm_process_group = nullptr; + } // Destroy the buffer manager m_memory_buffer = nullptr; @@ -1395,6 +1688,7 @@ void AtmosphereDriver::finalize ( /* inputs? */ ) { m_atm_comm.all_reduce(&my_mem_usage,&max_mem_usage,1,MPI_MAX); m_atm_logger->debug("[EAMxx::finalize] memory usage: " + std::to_string(max_mem_usage) + "MB"); #endif + m_atm_logger->flush(); m_ad_status = 0; diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index 0ba183cf74d3..bf694da3fb3e 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -2,6 +2,7 @@ #define SCREAM_ATMOSPHERE_DRIVER_HPP #include "control/surface_coupling_utils.hpp" +#include "control/intensive_observation_period.hpp" #include "share/field/field_manager.hpp" #include "share/grid/grids_manager.hpp" #include "share/util/scream_time_stamp.hpp" @@ -70,6 +71,9 @@ class AtmosphereDriver // Set AD params void init_scorpio (const int atm_id = 0); + // Setup IntensiveObservationPeriod + void setup_intensive_observation_period (); + // Create atm processes, without initializing them void create_atm_processes (); @@ -91,12 +95,24 @@ class AtmosphereDriver void setup_surface_coupling_processes() const; // Zero out precipitation flux - void reset_accummulated_fields(); + void reset_accumulated_fields(); // Create and add mass and energy conservation checks // and pass to m_atm_process_group. void setup_column_conservation_checks (); + // If TMS process exists, creates link to SHOC for applying + // tms' surface drag coefficient. + void setup_shoc_tms_links(); + + // Add column data to all pre/postcondition property checks + // for use in output. + void add_additional_column_data_to_property_checks (); + + void set_provenance_data (std::string caseid = "", + std::string hostname = "", + std::string username = ""); + // Load initial conditions for atm inputs void initialize_fields (); @@ -136,7 +152,6 @@ class AtmosphereDriver // NOTE: if already finalized, this is a no-op void finalize (); - field_mgr_ptr get_ref_grid_field_mgr () const; field_mgr_ptr get_field_mgr (const std::string& grid_name) const; // Get atmosphere time stamp @@ -192,6 +207,8 @@ class AtmosphereDriver std::shared_ptr m_surface_coupling_import_data_manager; std::shared_ptr m_surface_coupling_export_data_manager; + std::shared_ptr m_intensive_observation_period; + // This is the time stamp at the beginning of the time step. util::TimeStamp m_current_ts; diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp index ebbd993b4f94..dfdea16ae93f 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp @@ -64,12 +64,12 @@ void SurfaceCouplingExporter::set_grids(const std::shared_ptr("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name); create_helper_field("Sa_z", scalar2d_layout, grid_name); - create_helper_field("Sa_u", scalar2d_layout, grid_name); - create_helper_field("Sa_v", scalar2d_layout, grid_name); - create_helper_field("Sa_tbot", scalar2d_layout, grid_name); + create_helper_field("Sa_u", scalar2d_layout, grid_name); + create_helper_field("Sa_v", scalar2d_layout, grid_name); + create_helper_field("Sa_tbot", scalar2d_layout, grid_name); create_helper_field("Sa_ptem", scalar2d_layout, grid_name); - create_helper_field("Sa_pbot", scalar2d_layout, grid_name); - create_helper_field("Sa_shum", scalar2d_layout, grid_name); + create_helper_field("Sa_pbot", scalar2d_layout, grid_name); + create_helper_field("Sa_shum", scalar2d_layout, grid_name); create_helper_field("Sa_dens", scalar2d_layout, grid_name); create_helper_field("Sa_pslv", scalar2d_layout, grid_name); create_helper_field("Faxa_rainl", scalar2d_layout, grid_name); @@ -80,6 +80,7 @@ void SurfaceCouplingExporter::set_grids(const std::shared_ptr("fields"); + + EKAT_REQUIRE_MSG(export_from_file_params.isParameter("files"), + "Error! surface_coupling_exporter::init - prescribed_from_file does not have 'files' parameter."); + auto export_from_file_names = export_from_file_params.get("files"); + + bool are_fields_present = export_from_file_fields.size() > 0; + bool are_files_present = export_from_file_names.size() > 0; + EKAT_REQUIRE_MSG(are_files_present==are_fields_present, + "ERROR!! When prescribing export fields from file, you must provide both fields names and file name(s).\n"); + bool do_export_from_file = are_fields_present and are_files_present; + if (do_export_from_file) { + vos_type export_from_file_reg_names; + export_from_file_reg_names = export_from_file_fields; + // Check if alternative names have been provided. This is useful for source data files + // that don't use the conventional EAMxx ATM->SRFC variable names. + auto alt_names = export_from_file_params.get("fields_alt_name",{}); + for (auto entry : alt_names) { + ekat::strip(entry, ' '); // remove empty spaces in case user did `a : b` + auto tokens = ekat::split(entry,':'); + EKAT_REQUIRE_MSG(tokens.size()==2, + "Error! surface_coupling_exporter::init - expected 'EAMxx_var_name:FILE_var_name' entry in fields_alt_names, got '" + entry + "' instead.\n"); + auto it = ekat::find(export_from_file_fields,tokens[0]); + EKAT_REQUIRE_MSG(it!=export_from_file_fields.end(), + "Error! surface_coupling_exporter::init - LHS of entry '" + entry + "' in field_alt_names does not match a valid EAMxx field.\n"); + // Make sure that a user hasn't accidentally copy/pasted + auto chk = ekat::find(export_from_file_reg_names,tokens[1]); + EKAT_REQUIRE_MSG(chk==export_from_file_reg_names.end(), + "Error! surface_coupling_exporter::init - RHS of entry '" + entry + "' in field_alt_names has already been used for a different field.\n"); + auto idx = std::distance(export_from_file_fields.begin(),it); + export_from_file_reg_names[idx] = tokens[1]; + } + // Construct a time interpolation object + m_time_interp = util::TimeInterpolation(m_grid,export_from_file_names); + for (size_t ii=0; ii("fields"); auto export_constant_values = export_constant_params.get("values"); EKAT_REQUIRE_MSG(export_constant_fields.size()==export_constant_values.size(),"Error! surface_coupling_exporter::init - prescribed_constants 'fields' and 'values' are not the same size"); - if (export_constant_fields.size()>0) { + bool are_fields_present = export_constant_fields.size() > 0; + if (are_fields_present) { // Determine which fields need constants - for (int i=0; i=0,"Error! surface_coupling_exporter - The number of exports derived from EAMxx < 0, something must have gone wrong in assigning the types of exports for all variables."); // Perform initial export (if any are marked for export during initialization) @@ -250,8 +318,13 @@ void SurfaceCouplingExporter::run_impl (const double dt) void SurfaceCouplingExporter::do_export(const double dt, const bool called_during_initialization) { if (m_num_const_exports>0) { - set_constant_exports(dt,called_during_initialization); + set_constant_exports(); + } + + if (m_num_from_file_exports>0) { + set_from_file_exports(dt); } + if (m_num_from_model_exports>0) { compute_eamxx_exports(dt,called_during_initialization); } @@ -260,17 +333,32 @@ void SurfaceCouplingExporter::do_export(const double dt, const bool called_durin do_export_to_cpl(called_during_initialization); } // ========================================================================================= -void SurfaceCouplingExporter::set_constant_exports(const double dt, const bool called_during_initialization) +void SurfaceCouplingExporter::set_constant_exports() { // Cycle through those fields that will be set to a constant value: + int num_set = 0; // Checker to make sure we got all the fields we wanted. for (int i=0; i(); Kokkos::deep_copy(field_view,m_export_constants.at(fname)); + num_set++; } } - + // Gotta catch em all + EKAT_REQUIRE_MSG(num_set==m_num_const_exports,"ERROR! SurfaceCouplingExporter::set_constant_exports() - Number of fields set to a constant (" + std::to_string(num_set) +") doesn't match the number recorded at initialization (" + std::to_string(m_num_const_exports) +"). Something went wrong."); + +} +// ========================================================================================= +void SurfaceCouplingExporter::set_from_file_exports(const int dt) +{ + // Perform interpolation on the data with the latest timestamp + auto ts = timestamp(); + if (dt > 0) { + ts += dt; + } + m_time_interp.perform_time_interpolation(ts); + } // ========================================================================================= // This compute_eamxx_exports routine handles all export variables that are derived from the EAMxx state. @@ -370,7 +458,7 @@ void SurfaceCouplingExporter::compute_eamxx_exports(const double dt, const bool // Currently only needed for Sa_z, Sa_dens and Sa_pslv const bool calculate_z_vars = export_source(idx_Sa_z)==FROM_MODEL || export_source(idx_Sa_dens)==FROM_MODEL - || export_source(idx_Sa_pslv)==FROM_MODEL; + || export_source(idx_Sa_pslv)==FROM_MODEL; if (calculate_z_vars) { PF::calculate_dz(team, pseudo_density_i, p_mid_i, T_mid_i, qv_i, dz_i); team.team_barrier(); @@ -383,10 +471,10 @@ void SurfaceCouplingExporter::compute_eamxx_exports(const double dt, const bool // Set the values in the helper fields which correspond to the exported variables - if (export_source(idx_Sa_z)==FROM_MODEL) { + if (export_source(idx_Sa_z)==FROM_MODEL) { // Assugb to Sa_z const auto s_z_mid_i = ekat::scalarize(z_mid_i); - Sa_z(i) = s_z_mid_i(num_levs-1); + Sa_z(i) = s_z_mid_i(num_levs-1); } if (export_source(idx_Sa_u)==FROM_MODEL) { @@ -411,9 +499,9 @@ void SurfaceCouplingExporter::compute_eamxx_exports(const double dt, const bool Sa_pbot(i) = s_p_mid_i(num_levs-1); } - if (export_source(idx_Sa_shum)==FROM_MODEL) { + if (export_source(idx_Sa_shum)==FROM_MODEL) { const auto s_qv_i = ekat::scalarize(qv_i); - Sa_shum(i) = s_qv_i(num_levs-1); + Sa_shum(i) = s_qv_i(num_levs-1); } if (export_source(idx_Sa_dens)==FROM_MODEL) { @@ -446,6 +534,7 @@ void SurfaceCouplingExporter::compute_eamxx_exports(const double dt, const bool if (m_export_source_h(idx_Faxa_swvdf)==FROM_MODEL) { Kokkos::deep_copy(Faxa_swvdf, sfc_flux_dif_vis); } if (m_export_source_h(idx_Faxa_swnet)==FROM_MODEL) { Kokkos::deep_copy(Faxa_swnet, sfc_flux_sw_net); } if (m_export_source_h(idx_Faxa_lwdn )==FROM_MODEL) { Kokkos::deep_copy(Faxa_lwdn, sfc_flux_lw_dn); } + } // ========================================================================================= void SurfaceCouplingExporter::do_export_to_cpl(const bool called_during_initialization) @@ -479,7 +568,7 @@ void SurfaceCouplingExporter::do_export_to_cpl(const bool called_during_initiali // ========================================================================================= void SurfaceCouplingExporter::finalize_impl() { - + // Nothing to do } // ========================================================================================= } // namespace scream diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp index 285401d1193b..d8ccf5862f3c 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp @@ -1,14 +1,15 @@ #ifndef SCREAM_EXPORTER_HPP #define SCREAM_EXPORTER_HPP +#include "surface_coupling_utils.hpp" + #include "share/atm_process/atmosphere_process.hpp" -#include "ekat/ekat_parameter_list.hpp" #include "share/util/scream_common_physics_functions.hpp" +#include "share/util/eamxx_time_interpolation.hpp" #include "share/atm_process/ATMBufferManager.hpp" #include "share/atm_process/SCDataManager.hpp" -#include "surface_coupling_utils.hpp" - +#include #include namespace scream @@ -79,12 +80,12 @@ class SurfaceCouplingExporter : public AtmosphereProcess // which do not have valid entries. void do_export(const double dt, const bool called_during_initialization=false); // Main export routine void compute_eamxx_exports(const double dt, const bool called_during_initialization=false); // Export vars are derived from eamxx state - void set_constant_exports(const double dt, const bool called_during_initialization=false); // Export vars are set to a constant + void set_constant_exports(); // Export vars are set to a constant + void set_from_file_exports(const int dt); // Export vars are set by interpolation of data from files void do_export_to_cpl(const bool called_during_initialization=false); // Finish export by copying data to cpl structures. // Take and store data from SCDataManager void setup_surface_coupling_data(const SCDataManager &sc_data_manager); - protected: // The three main overrides for the subcomponent @@ -123,12 +124,16 @@ class SurfaceCouplingExporter : public AtmosphereProcess Int m_num_cpl_exports; // Number of exports from EAMxx and how they will be handled - Int m_num_scream_exports; + int m_num_scream_exports; view_1d m_export_source; view_1d m_export_source_h; std::map m_export_constants; int m_num_from_model_exports=0; int m_num_const_exports=0; + // For exporting from file + int m_num_from_file_exports=0; + util::TimeInterpolation m_time_interp; + std::vector m_export_from_file_field_names; // Views storing a 2d array with dims (num_cols,num_fields) for cpl export data. // The field idx strides faster, since that's what mct does (so we can "view" the @@ -137,7 +142,8 @@ class SurfaceCouplingExporter : public AtmosphereProcess uview_2d m_cpl_exports_view_h; // Array storing the field names for exports - name_t* m_export_field_names; + name_t* m_export_field_names; + std::vector m_export_field_names_vector; // Views storing information for each export uview_1d m_cpl_indices_view; diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 828bc4a1abfb..1bf32b924215 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -1,6 +1,7 @@ #include "atmosphere_surface_coupling_importer.hpp" #include "share/property_checks/field_within_interval_check.hpp" +#include "physics/share/physics_constants.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_units.hpp" @@ -24,7 +25,7 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptrname(); m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank - + // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. auto Qunit = kg/kg; @@ -130,7 +131,7 @@ void SurfaceCouplingImporter::initialize_impl (const RunType /* run_type */) add_postcondition_check(get_field_out("sfc_alb_dif_vis"),m_grid,0.0,1.0,true); add_postcondition_check(get_field_out("sfc_alb_dif_nir"),m_grid,0.0,1.0,true); - // Perform initial import (if any are marked for import during initialization) + // Perform initial import (if any are marked for import during initialization) if (any_initial_imports) do_import(true); } // ========================================================================================= @@ -168,6 +169,70 @@ void SurfaceCouplingImporter::do_import(const bool called_during_initialization) info.data[offset] = cpl_imports_view_d(icol,info.cpl_indx)*info.constant_multiple; } }); + + // If IOP is defined, potentially overwrite imports with data from IOP file + if (m_intensive_observation_period) { + overwrite_iop_imports(called_during_initialization); + } +} +// ========================================================================================= +void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_initialization) +{ + using policy_type = KokkosTypes::RangePolicy; + using C = physics::Constants; + + const auto& iop = m_intensive_observation_period; + + const auto has_lhflx = iop->has_iop_field("lhflx"); + const auto has_shflx = iop->has_iop_field("shflx"); + const auto has_Tg = iop->has_iop_field("Tg"); + + static constexpr Real latvap = C::LatVap; + static constexpr Real stebol = C::stebol; + + const auto& col_info_h = m_column_info_h; + const auto& col_info_d = m_column_info_d; + + for (int ifield=0; ifieldget_iop_field("lhflx"); + f.sync_to_host(); + col_val = f.get_view()()/latvap; + } else if (fname == "surf_sens_flux" && has_shflx) { + const auto f = iop->get_iop_field("shflx"); + f.sync_to_host(); + col_val = f.get_view()(); + } else if (fname == "surf_radiative_T" && has_Tg) { + const auto f = iop->get_iop_field("Tg"); + f.sync_to_host(); + col_val = f.get_view()(); + } else if (fname == "surf_lw_flux_up" && has_Tg) { + const auto f = iop->get_iop_field("Tg"); + f.sync_to_host(); + col_val = stebol*std::pow(f.get_view()(), 4); + } else { + // If import field doesn't satisify above, skip + continue; + } + + // Overwrite iop imports with col_val for each column + auto policy = policy_type(0, m_num_cols); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const int& icol) { + const auto& info_d = col_info_d(ifield); + const auto offset = icol*info_d.col_stride + info_d.col_offset; + info_d.data[offset] = col_val; + }); + } } // ========================================================================================= void SurfaceCouplingImporter::finalize_impl() diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp index 14a29f373d4c..5884c9d40af2 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp @@ -5,6 +5,8 @@ #include "ekat/ekat_parameter_list.hpp" #include "share/atm_process/SCDataManager.hpp" +#include "control/intensive_observation_period.hpp" + #include "surface_coupling_utils.hpp" #include @@ -33,7 +35,8 @@ class SurfaceCouplingImporter : public AtmosphereProcess template using uview_2d = Unmanaged>; - using name_t = char[32]; + using name_t = char[32]; + using iop_ptr = std::shared_ptr; // Constructors SurfaceCouplingImporter (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -58,6 +61,12 @@ class SurfaceCouplingImporter : public AtmosphereProcess // Take and store data from SCDataManager void setup_surface_coupling_data(const SCDataManager &sc_data_manager); + // Overwrite imports for IOP cases with IOP file surface data + void overwrite_iop_imports (const bool called_during_initialization); + + void set_intensive_observation_period (const iop_ptr& iop) { + m_intensive_observation_period = iop; + } protected: // The three main overrides for the subcomponent @@ -66,7 +75,7 @@ class SurfaceCouplingImporter : public AtmosphereProcess void finalize_impl (); // Keep track of field dimensions - Int m_num_cols; + Int m_num_cols; // Number of fields in cpl data Int m_num_cpl_imports; @@ -93,10 +102,11 @@ class SurfaceCouplingImporter : public AtmosphereProcess view_1d m_column_info_d; decltype(m_column_info_d)::HostMirror m_column_info_h; + // Intensive observation period object. + iop_ptr m_intensive_observation_period; // The grid is needed for property checks std::shared_ptr m_grid; - }; // class SurfaceCouplingImporter } // namespace scream diff --git a/components/eamxx/src/control/fvphyshack.cpp b/components/eamxx/src/control/fvphyshack.cpp deleted file mode 100644 index 515f66edd537..000000000000 --- a/components/eamxx/src/control/fvphyshack.cpp +++ /dev/null @@ -1,3 +0,0 @@ -namespace scream { -bool fvphyshack; -} diff --git a/components/eamxx/src/control/fvphyshack.hpp b/components/eamxx/src/control/fvphyshack.hpp deleted file mode 100644 index a1c57bbd34df..000000000000 --- a/components/eamxx/src/control/fvphyshack.hpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "ekat/ekat_parameter_list.hpp" - -namespace scream { -extern bool fvphyshack; -void fv_phys_rrtmgp_active_gases_init(const ekat::ParameterList& p); -void fv_phys_rrtmgp_active_gases_set_restart(const bool restart); -} diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp new file mode 100644 index 000000000000..f4ab466c17ba --- /dev/null +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -0,0 +1,849 @@ +#include "control/intensive_observation_period.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/util/scream_vertical_interpolation.hpp" + +#include "ekat/ekat_assert.hpp" +#include "ekat/util/ekat_lin_interp.hpp" + +#include "pio.h" + +#include + +// Extend ekat mpi type for pairs, +// used for reduction of type MPI_MINLOC. +namespace ekat { +#ifdef SCREAM_DOUBLE_PRECISION + template<> + MPI_Datatype get_mpi_type> () { + return MPI_DOUBLE_INT; + } +#else + template<> + MPI_Datatype get_mpi_type> () { + return MPI_FLOAT_INT; + } +#endif +} + +namespace scream { +namespace control { + +// Helper functions for reading data from .nc file to support +// cases not currently supported in EAMxx scorpio interface. +namespace { +// Read the value of a dimensionless variable from file. +template +void read_dimensionless_variable_from_file(const std::string& filename, + const std::string& varname, + T* value) +{ + EKAT_REQUIRE_MSG(scorpio::has_variable(filename,varname), + "Error! IOP file does not have variable "+varname+".\n"); + + int ncid, varid, err1, err2; + bool was_open = scorpio::is_file_open_c2f(filename.c_str(),-1); + if (not was_open) { + scorpio::register_file(filename,scorpio::FileMode::Read); + } + ncid = scorpio::get_file_ncid_c2f (filename.c_str()); + err1 = PIOc_inq_varid(ncid,varname.c_str(),&varid); + EKAT_REQUIRE_MSG(err1==PIO_NOERR, + "Error! Something went wrong while retrieving variable id.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - pio error: " + std::to_string(err1) + "\n"); + + err2 = PIOc_get_var(ncid, varid, value); + EKAT_REQUIRE_MSG(err2==PIO_NOERR, + "Error! Something went wrong while retrieving variable.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - pio error: " + std::to_string(err2) + "\n"); + + if (not was_open) { + scorpio::eam_pio_closefile(filename); + } +} + +// Read variable with arbitrary number of dimensions from file. +template +void read_variable_from_file(const std::string& filename, + const std::string& varname, + const std::string& vartype, + const std::vector& dimnames, + const int time_idx, + T* data) +{ + EKAT_REQUIRE_MSG(scorpio::has_variable(filename,varname), + "Error! IOP file does not have variable "+varname+".\n"); + + // Compute total size of data to read + int data_size = 1; + for (auto dim : dimnames) { + const auto dim_len = scorpio::get_dimlen(filename, dim); + data_size *= dim_len; + } + + // Read into data + scorpio::register_file(filename, scorpio::FileMode::Read); + std::string io_decomp_tag = varname+","+filename; + scorpio::register_variable(filename, varname, varname, dimnames, vartype, io_decomp_tag); + std::vector dof_offsets(data_size); + std::iota(dof_offsets.begin(), dof_offsets.end(), 0); + scorpio::set_dof(filename, varname, dof_offsets.size(), dof_offsets.data()); + scorpio::set_decomp(filename); + scorpio::grid_read_data_array(filename, varname, time_idx, data, data_size); + scorpio::eam_pio_closefile(filename); +} +} + +IntensiveObservationPeriod:: +IntensiveObservationPeriod(const ekat::Comm& comm, + const ekat::ParameterList& params, + const util::TimeStamp& run_t0, + const int model_nlevs, + const Field& hyam, + const Field& hybm) +{ + m_comm = comm; + m_params = params; + EKAT_REQUIRE_MSG(m_params.get("doubly_periodic_mode", false), + "Error! Currently doubly_periodic_mode is the only use case for " + "intensive observation period files.\n"); + + EKAT_REQUIRE_MSG(m_params.isParameter("target_latitude") && m_params.isParameter("target_longitude"), + "Error! Using intensive observation period files requires " + "target_latitude and target_longitude be gives as parameters in " + "\"intensive_observation_period_options\" in the input yaml file.\n"); + const auto target_lat = m_params.get("target_latitude"); + const auto target_lon = m_params.get("target_longitude"); + EKAT_REQUIRE_MSG(-90 <= target_lat and target_lat <= 90, + "Error! IOP target_lat="+std::to_string(target_lat)+" outside of expected range [-90, 90].\n"); + EKAT_REQUIRE_MSG(0 <= target_lon and target_lon <= 360, + "Error! IOP target_lat="+std::to_string(target_lon)+" outside of expected range [0, 360].\n"); + + // Set defaults for some parameters + if (not m_params.isParameter("iop_srf_prop")) m_params.set("iop_srf_prop", false); + if (not m_params.isParameter("iop_dosubsidence")) m_params.set("iop_dosubsidence", false); + if (not m_params.isParameter("iop_coriolis")) m_params.set("iop_coriolis", false); + if (not m_params.isParameter("iop_nudge_tq")) m_params.set("iop_nudge_tq", false); + if (not m_params.isParameter("iop_nudge_uv")) m_params.set("iop_nudge_uv", false); + if (not m_params.isParameter("iop_nudge_tq_low")) m_params.set("iop_nudge_tq_low", 1050); + if (not m_params.isParameter("iop_nudge_tq_high")) m_params.set("iop_nudge_tq_high", 0); + if (not m_params.isParameter("iop_nudge_tscale")) m_params.set("iop_nudge_tscale", 10800); + if (not m_params.isParameter("zero_non_iop_tracers")) m_params.set("zero_non_iop_tracers", false); + + // Use IOP file to initialize parameters + // and timestepping information + initialize_iop_file(run_t0, model_nlevs, hyam, hybm); +} + +void IntensiveObservationPeriod:: +initialize_iop_file(const util::TimeStamp& run_t0, + int model_nlevs, + const Field& hyam, + const Field& hybm) +{ + EKAT_REQUIRE_MSG(m_params.isParameter("iop_file"), + "Error! Using IOP requires defining an iop_file parameter.\n"); + + const auto iop_file = m_params.get("iop_file"); + + // Lambda for allocating space and storing information for potential iop fields. + // Inputs: + // - varnames: Vector of possible variable names in the iop file. + // First entry will be the variable name used when accessing in class + // - fl: IOP field layout (acceptable ranks: 0, 1) + // - srf_varname: Name of surface variable potentially in iop file associated with iop variable. + auto setup_iop_field = [&, this] (const vos& varnames, + const FieldLayout& fl, + const std::string& srf_varname = "none") { + EKAT_REQUIRE_MSG(fl.rank() == 0 || fl.rank() == 1, + "Error! IOP fields must have rank 0 or 1. " + "Attempting to setup "+varnames[0]+" with rank " + +std::to_string(fl.rank())+".\n"); + + // Check if var exists in IOP file. Some variables will + // need to check alternate names. + const auto iop_varname = varnames[0]; + bool has_var = false; + std::string file_varname = ""; + for (auto varname : varnames) { + if (scorpio::has_variable(iop_file, varname)) { + has_var = true; + file_varname = varname; + break; + }; + } + if (has_var) { + // Store if iop file has a different varname than the iop field + if (iop_varname != file_varname) m_iop_file_varnames.insert({iop_varname, file_varname}); + // Store if variable contains a surface value in iop file + if (scorpio::has_variable(iop_file, srf_varname)) { + m_iop_field_surface_varnames.insert({iop_varname, srf_varname}); + } + + // Allocate field for variable + FieldIdentifier fid(iop_varname, fl, ekat::units::Units::nondimensional(), ""); + const auto field_rank = fl.rank(); + EKAT_REQUIRE_MSG(field_rank <= 1, + "Error! Unexpected field rank "+std::to_string(field_rank)+" for iop file fields.\n"); + Field field(fid); + field.allocate_view(); + m_iop_fields.insert({iop_varname, field}); + } + }; + + // Check if the following variables exist in the iop file + + // Scalar data + FieldLayout fl_scalar({},{}); // Zero dim fields used for iop file scalars + setup_iop_field({"Ps"}, fl_scalar); + setup_iop_field({"Tg"}, fl_scalar); + setup_iop_field({"lhflx", "lh"}, fl_scalar); + setup_iop_field({"shflx", "sh"}, fl_scalar); + + // Level data + FieldLayout fl_vector({FieldTag::LevelMidPoint}, {model_nlevs}); + setup_iop_field({"T"}, fl_vector, "Tsair"); + setup_iop_field({"q"}, fl_vector, "qsrf"); + setup_iop_field({"cld"}, fl_vector); + setup_iop_field({"clwp"}, fl_vector); + setup_iop_field({"divq"}, fl_vector, "divqsrf"); + setup_iop_field({"vertdivq"}, fl_vector, "vertdivqsrf"); + setup_iop_field({"NUMLIQ"}, fl_vector); + setup_iop_field({"CLDLIQ"}, fl_vector); + setup_iop_field({"CLDICE"}, fl_vector); + setup_iop_field({"NUMICE"}, fl_vector); + setup_iop_field({"divu"}, fl_vector, "divusrf"); + setup_iop_field({"divv"}, fl_vector, "divvsrf"); + setup_iop_field({"divT"}, fl_vector, "divtsrf"); + setup_iop_field({"vertdivT"}, fl_vector, "vertdivTsrf"); + setup_iop_field({"divT3d"}, fl_vector, "divT3dsrf"); + setup_iop_field({"u"}, fl_vector, "usrf"); + setup_iop_field({"u_ls"}, fl_vector, "usrf"); + setup_iop_field({"v"}, fl_vector, "vsrf"); + setup_iop_field({"v_ls"}, fl_vector, "vsrf"); + setup_iop_field({"Q1"}, fl_vector); + setup_iop_field({"Q2"}, fl_vector); + setup_iop_field({"omega"}, fl_vector, "Ptend"); + + // Make sure Ps, T, and q are defined in the iop file + EKAT_REQUIRE_MSG(has_iop_field("Ps"), + "Error! Using IOP file requires variable \"Ps\".\n"); + EKAT_REQUIRE_MSG(has_iop_field("T"), + "Error! Using IOP file requires variable \"T\".\n"); + EKAT_REQUIRE_MSG(has_iop_field("q"), + "Error! Using IOP file requires variable \"q\".\n"); + + // Initialize time information + int bdate; + std::string bdate_name; + if (scorpio::has_variable(iop_file, "bdate")) bdate_name = "bdate"; + else if (scorpio::has_variable(iop_file, "basedate")) bdate_name = "basedate"; + else if (scorpio::has_variable(iop_file, "nbdate")) bdate_name = "nbdate"; + else EKAT_ERROR_MSG("Error! No valid name for bdate in "+iop_file+".\n"); + read_dimensionless_variable_from_file(iop_file, bdate_name, &bdate); + + int yr=bdate/10000; + int mo=(bdate/100) - yr*100; + int day=bdate - (yr*10000+mo*100); + m_time_info.iop_file_begin_time = util::TimeStamp(yr,mo,day,0,0,0); + + std::string time_dimname; + if (scorpio::has_dim(iop_file, "time")) time_dimname = "time"; + else if (scorpio::has_dim(iop_file, "tsec")) time_dimname = "tsec"; + else EKAT_ERROR_MSG("Error! No valid dimension for tsec in "+iop_file+".\n"); + const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); + m_time_info.iop_file_times_in_sec = + decltype(m_time_info.iop_file_times_in_sec)("iop_file_times", ntimes); + read_variable_from_file(iop_file, "tsec", "int", {time_dimname}, -1, + m_time_info.iop_file_times_in_sec.data()); + + // Check that lat/lon from iop file match the targets in parameters. Note that + // longitude may be negtive in the iop file, we convert to positive before checking. + const auto nlats = scorpio::get_dimlen(iop_file, "lat"); + const auto nlons = scorpio::get_dimlen(iop_file, "lon"); + EKAT_REQUIRE_MSG(nlats==1 and nlons==1, "Error! IOP data file requires a single lat/lon pair.\n"); + Real iop_file_lat, iop_file_lon; + read_variable_from_file(iop_file, "lat", "real", {"lat"}, -1, &iop_file_lat); + read_variable_from_file(iop_file, "lon", "real", {"lon"}, -1, &iop_file_lon); + EKAT_REQUIRE_MSG(iop_file_lat == m_params.get("target_latitude"), + "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); + EKAT_REQUIRE_MSG(std::fmod(iop_file_lon + 360, 360) == m_params.get("target_longitude"), + "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); + + // Store iop file pressure as helper field with dimension lev+1. + // Load the first lev entries from iop file, the lev+1 entry will + // be set when reading iop data. + EKAT_REQUIRE_MSG(scorpio::has_variable(iop_file, "lev"), + "Error! Using IOP file requires variable \"lev\".\n"); + const auto file_levs = scorpio::get_dimlen(iop_file, "lev"); + FieldIdentifier fid("iop_file_pressure", + FieldLayout({FieldTag::LevelMidPoint}, {file_levs+1}), + ekat::units::Units::nondimensional(), + ""); + Field iop_file_pressure(fid); + iop_file_pressure.allocate_view(); + auto data = iop_file_pressure.get_view().data(); + read_variable_from_file(iop_file, "lev", "real", {"lev"}, -1, data); + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilevname(); + + // Create io grid if doesn't exist + if (m_io_grids.count(grid_name) == 0) { + // IO grid needs to have ncol dimension equal to the IC/topo file + const auto nc_file_ncols = scorpio::get_dimlen(file_name, "ncol"); + const auto nlevs = grid->get_num_vertical_levels(); + m_io_grids[grid_name] = create_point_grid(grid_name, + nc_file_ncols, + nlevs, + m_comm); + } + + // Store closest lat/lon info for this grid if doesn't exist + if (m_lat_lon_info.count(grid_name) == 0) { + const auto& io_grid = m_io_grids[grid_name]; + + // Create lat/lon fields + const auto ncols = io_grid->get_num_local_dofs(); + std::vector fields; + + FieldIdentifier lat_fid("lat", + FieldLayout({FieldTag::Column},{ncols}), + ekat::units::Units::nondimensional(), + grid_name); + Field lat_f(lat_fid); + lat_f.allocate_view(); + fields.push_back(lat_f); + + FieldIdentifier lon_fid("lon", + FieldLayout({FieldTag::Column},{ncols}), + ekat::units::Units::nondimensional(), + grid_name); + Field lon_f(lon_fid); + lon_f.allocate_view(); + fields.push_back(lon_f); + + // Read from file + AtmosphereInput file_reader(file_name, io_grid, fields); + file_reader.read_variables(); + file_reader.finalize(); + + // Find column index of closest lat/lon to target_lat/lon params + auto lat_v = fields[0].get_view(); + auto lon_v = fields[1].get_view(); + const auto target_lat = m_params.get("target_latitude"); + const auto target_lon = m_params.get("target_longitude"); + using minloc_t = Kokkos::MinLoc; + using minloc_value_t = typename minloc_t::value_type; + minloc_value_t minloc; + Kokkos::parallel_reduce(ncols, KOKKOS_LAMBDA (int icol, minloc_value_t& result) { + auto dist = std::abs(lat_v(icol)-target_lat)+std::abs(lon_v(icol)-target_lon); + if(dist min_dist_and_rank = {minloc.val, my_rank}; + m_comm.all_reduce>(&min_dist_and_rank, 1, MPI_MINLOC); + + // Broadcast closest lat/lon values to all ranks + const auto lat_v_h = lat_f.get_view(); + const auto lon_v_h = lon_f.get_view(); + auto local_column_idx = minloc.loc; + auto min_dist_rank = min_dist_and_rank.second; + Real lat_lon_vals[2]; + if (my_rank == min_dist_rank) { + lat_lon_vals[0] = lat_v_h(local_column_idx); + lat_lon_vals[1] = lon_v_h(local_column_idx); + } + m_comm.broadcast(lat_lon_vals, 2, min_dist_rank); + + // Set local_column_idx=-1 for mpi ranks not containing minimum lat/lon distance + if (my_rank != min_dist_rank) local_column_idx = -1; + + // Store closest lat/lon info for this grid, used later when reading ICs + m_lat_lon_info[grid_name] = ClosestLatLonInfo{lat_lon_vals[0], lat_lon_vals[1], min_dist_rank, local_column_idx}; + } +} + +void IntensiveObservationPeriod:: +read_fields_from_file_for_iop (const std::string& file_name, + const vos& field_names_nc, + const vos& field_names_eamxx, + const util::TimeStamp& initial_ts, + const field_mgr_ptr field_mgr) +{ + const auto dummy_units = ekat::units::Units::nondimensional(); + + EKAT_REQUIRE_MSG(field_names_nc.size()==field_names_eamxx.size(), + "Error! Field name arrays must have same size.\n"); + + if (field_names_nc.size()==0) { + return; + } + + const auto& grid_name = field_mgr->get_grid()->name(); + EKAT_REQUIRE_MSG(m_io_grids.count(grid_name) > 0, + "Error! Attempting to read IOP initial conditions on " + +grid_name+" grid, but m_io_grid entry has not been created.\n"); + EKAT_REQUIRE_MSG(m_lat_lon_info.count(grid_name) > 0, + "Error! Attempting to read IOP initial conditions on " + +grid_name+" grid, but m_lat_lon_info entry has not been created.\n"); + + auto io_grid = m_io_grids[grid_name]; + if (grid_name=="Physics GLL" && scorpio::has_dim(file_name,"ncol_d")) { + // If we are on GLL grid, and nc file contains "ncol_d" dimension, + // we need to reset COL dim tag + using namespace ShortFieldTagsNames; + auto grid = io_grid->clone(io_grid->name(),true); + grid->reset_field_tag_name(COL,"ncol_d"); + io_grid = grid; + } + + // Create vector of fields with correct dimensions to read from file + std::vector io_fields; + for (size_t i=0; iget_field(eamxx_name).alias(nc_name) + : + field_mgr->get_field(eamxx_name); + auto fm_fid = fm_field.get_header().get_identifier(); + EKAT_REQUIRE_MSG(fm_fid.get_layout().tag(0)==FieldTag::Column, + "Error! IOP inputs read from IC/topo file must have Column " + "as first dim tag.\n"); + + // Set first dimension to match input file + auto dims = fm_fid.get_layout().dims(); + dims[0] = io_grid->get_num_local_dofs(); + FieldLayout io_fl(fm_fid.get_layout().tags(), dims); + FieldIdentifier io_fid(fm_fid.name(), io_fl, fm_fid.get_units(), io_grid->name()); + Field io_field(io_fid); + io_field.allocate_view(); + io_fields.push_back(io_field); + } + + // Read data from file + AtmosphereInput file_reader(file_name,io_grid,io_fields); + file_reader.read_variables(); + file_reader.finalize(); + + // For each field, broadcast data from closest lat/lon column to all processors + // and copy data into each field's column in the field manager. + for (size_t i=0; iget_field(fname); + + // Create a temporary field to store the data from the + // single column of the closest lat/lon pair + const auto io_fid = io_field.get_header().get_identifier(); + FieldLayout col_data_fl = io_fid.get_layout().strip_dim(0); + FieldIdentifier col_data_fid("col_data", col_data_fl, dummy_units, ""); + Field col_data(col_data_fid); + col_data.allocate_view(); + + // MPI rank with closest column index store column data + const auto mpi_rank_with_col = m_lat_lon_info[grid_name].mpi_rank_of_closest_column; + if (m_comm.rank() == mpi_rank_with_col) { + const auto col_idx_with_data = m_lat_lon_info[grid_name].local_column_index_of_closest_column; + col_data.deep_copy(io_field.subfield(0,col_idx_with_data)); + } + + // Broadcast column data to all other ranks + const auto col_size = col_data.get_header().get_identifier().get_layout().size(); + m_comm.broadcast(col_data.get_internal_view_data(), col_size, mpi_rank_with_col); + + // Copy column data to all columns in field manager field + const auto ncols = fm_field.get_header().get_identifier().get_layout().dim(0); + for (auto icol=0; icol(col_data); + } + + // Sync fields to device + fm_field.sync_to_dev(); + + // Set the initial time stamp on FM fields + fm_field.get_header().get_tracking().update_time_stamp(initial_ts); + } +} + +void IntensiveObservationPeriod:: +read_iop_file_data (const util::TimeStamp& current_ts) +{ + const auto iop_file = m_params.get("iop_file"); + const auto iop_file_time_idx = m_time_info.get_iop_file_time_idx(current_ts); + + // Sanity check + EKAT_REQUIRE_MSG(iop_file_time_idx >= m_time_info.time_idx_of_current_data, + "Error! Attempting to read previous iop file data time index.\n"); + + // If we are still in the time interval as the previous read from iop file, + // there is no need to reload data. Return early + if (iop_file_time_idx == m_time_info.time_idx_of_current_data) return; + + const auto file_levs = scorpio::get_dimlen(iop_file, "lev"); + const auto iop_file_pressure = m_helper_fields["iop_file_pressure"]; + const auto model_pressure = m_helper_fields["model_pressure"]; + const auto surface_pressure = m_iop_fields["Ps"]; + + // Loop through iop fields, if rank 1 fields exist we need to + // gather information for vertically interpolating views + bool has_level_data = false; + for (auto& it : m_iop_fields) { + if (it.second.rank() == 1) { + has_level_data = true; + break; + } + } + + // Compute values and indices associate with pressure for interpolating data (if necessary). + int adjusted_file_levs; + int iop_file_start; + int iop_file_end; + int model_start; + int model_end; + if (has_level_data) { + // Load surface pressure (Ps) from iop file + auto ps_data = surface_pressure.get_view().data(); + read_variable_from_file(iop_file, "Ps", "real", {"lon","lat"}, iop_file_time_idx, ps_data); + surface_pressure.sync_to_dev(); + + // Pre-process file pressures, store number of file levels + // where the last level is the first level equal to surface pressure. + const auto iop_file_pres_v = iop_file_pressure.get_view(); + // Sanity check + EKAT_REQUIRE_MSG(file_levs+1 == iop_file_pressure.get_header().get_identifier().get_layout().dim(0), + "Error! Unexpected size for helper field \"iop_file_pressure\"\n"); + const auto& Ps = surface_pressure.get_view(); + Kokkos::parallel_reduce(file_levs+1, KOKKOS_LAMBDA (const int ilev, int& lmin) { + if (ilev == file_levs) { + // Add surface pressure to last iop file pressure entry + iop_file_pres_v(ilev) = Ps()/100; + } + if (iop_file_pres_v(ilev) > Ps()/100) { + // Set upper bound on pressure values + iop_file_pres_v(ilev) = Ps()/100; + } + if (iop_file_pres_v(ilev) == Ps()/100) { + // Find minimum number of levels where the final + // level would contain the largest value. + if (ilev < lmin) lmin = ilev+1; + } + }, Kokkos::Min(adjusted_file_levs)); + + EKAT_REQUIRE_MSG(adjusted_file_levs > 1, + "Error! Pressures in iop file "+iop_file+" is are inccorrectly set. " + "Surface pressure \"Ps\" (converted to millibar) should be greater " + "than at least the 1st entry in midpoint pressures \"lev\".\n"); + + // Compute model pressure levels + const auto model_pres_v = model_pressure.get_view(); + const auto model_nlevs = model_pressure.get_header().get_identifier().get_layout().dim(0); + const auto hyam_v = m_helper_fields["hyam"].get_view(); + const auto hybm_v = m_helper_fields["hybm"].get_view(); + Kokkos::parallel_for(model_nlevs, KOKKOS_LAMBDA (const int ilev) { + model_pres_v(ilev) = 1000*hyam_v(ilev) + Ps()*hybm_v(ilev)/100; + }); + + // Find file pressure levels just outside the range of model pressure levels + Kokkos::parallel_reduce(adjusted_file_levs, KOKKOS_LAMBDA (const int& ilev, int& lmax, int& lmin) { + if (iop_file_pres_v(ilev) <= model_pres_v(0) && ilev > lmax) { + lmax = ilev; + } + if (iop_file_pres_v(ilev) >= model_pres_v(model_nlevs-1) && ilev+1 < lmin) { + lmin = ilev+1; + } + }, + Kokkos::Max(iop_file_start), + Kokkos::Min(iop_file_end)); + + // Find model pressure levels just inside range of file pressure levels + Kokkos::parallel_reduce(model_nlevs, KOKKOS_LAMBDA (const int& ilev, int& lmin, int& lmax) { + if (model_pres_v(ilev) >= iop_file_pres_v(iop_file_start) && ilev < lmin) { + lmin = ilev; + } + if (model_pres_v(ilev) <= iop_file_pres_v(iop_file_end-1) && ilev+1 > lmax) { + lmax = ilev+1; + } + }, + Kokkos::Min(model_start), + Kokkos::Max(model_end)); + } + + // Loop through fields and store data from file + for (auto& it : m_iop_fields) { + auto fname = it.first; + auto field = it.second; + + // File may use different varname than IOP class + auto file_varname = (m_iop_file_varnames.count(fname) > 0) ? m_iop_file_varnames[fname] : fname; + + if (field.rank()==0) { + // For scalar data, read iop file variable directly into field data + auto data = field.get_view().data(); + read_variable_from_file(iop_file, file_varname, "real", {"lon","lat"}, iop_file_time_idx, data); + field.sync_to_dev(); + } else if (field.rank()==1) { + // Create temporary fields for reading iop file variables. We use + // adjusted_file_levels (computed above) which contains an unset + // value for surface. + FieldIdentifier fid(file_varname+"_iop_file", + FieldLayout({FieldTag::LevelMidPoint}, + {adjusted_file_levs}), + ekat::units::Units::nondimensional(), + ""); + Field iop_file_field(fid); + iop_file_field.allocate_view(); + + // Read data from iop file. + std::vector data(file_levs); + read_variable_from_file(iop_file, file_varname, "real", {"lon","lat","lev"}, iop_file_time_idx, data.data()); + + // Copy first adjusted_file_levs-1 values to field + auto iop_file_v_h = iop_file_field.get_view(); + for (int ilev=0; ilev0; + if (has_srf) { + const auto srf_varname = m_iop_field_surface_varnames[fname]; + read_variable_from_file(iop_file, srf_varname, "real", {"lon","lat"}, iop_file_time_idx, &iop_file_v_h(adjusted_file_levs-1)); + } else { + // No surface value exists, compute surface value + const auto dx = iop_file_v_h(adjusted_file_levs-2) - iop_file_v_h(adjusted_file_levs-3); + if (dx == 0) iop_file_v_h(adjusted_file_levs-1) = iop_file_v_h(adjusted_file_levs-2); + else { + const auto iop_file_pres_v_h = iop_file_pressure.get_view(); + const auto dy = iop_file_pres_v_h(adjusted_file_levs-2) - iop_file_pres_v_h(adjusted_file_levs-3); + const auto scale = dy/dx; + + iop_file_v_h(adjusted_file_levs-1) = + (iop_file_pres_v_h(adjusted_file_levs-1)-iop_file_pres_v_h(adjusted_file_levs-2))/scale + + iop_file_v_h(adjusted_file_levs-2); + } + } + iop_file_field.sync_to_dev(); + + // Vertically interpolate iop file data to iop fields. + // Note: ekat lininterp requires packs. Use 1d packs here. + // TODO: allow for nontrivial packsize. + const auto iop_file_pres_v = iop_file_pressure.get_view(); + const auto model_pres_v = model_pressure.get_view(); + const auto iop_file_v = iop_file_field.get_view(); + auto iop_field_v = field.get_view(); + + const auto nlevs_input = iop_file_end - iop_file_start; + const auto nlevs_output = model_end - model_start; + const auto total_nlevs = field.get_header().get_identifier().get_layout().dim(0); + + ekat::LinInterp vert_interp(1, nlevs_input, nlevs_output); + const auto policy = ESU::get_default_team_policy(1, total_nlevs); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA (const KT::MemberType& team) { + const auto x_src = Kokkos::subview(iop_file_pres_v, Kokkos::pair(iop_file_start,iop_file_end)); + const auto x_tgt = Kokkos::subview(model_pres_v, Kokkos::pair(model_start,model_end)); + const auto input = Kokkos::subview(iop_file_v, Kokkos::pair(iop_file_start,iop_file_end)); + const auto output = Kokkos::subview(iop_field_v, Kokkos::pair(model_start,model_end)); + + vert_interp.setup(team, x_src, x_tgt); + vert_interp.lin_interp(team, x_src, x_tgt, input, output); + }); + Kokkos::fence(); + + // For certain fields we need to make sure to fill in the ends of + // the interpolated region with the value at model_start/model_end + if (fname == "T" || fname == "q" || fname == "u" || + fname == "u_ls" || fname == "v" || fname == "v_ls") { + if (model_start > 0) { + Kokkos::parallel_for(Kokkos::RangePolicy<>(0, model_start), + KOKKOS_LAMBDA (const int ilev) { + iop_field_v(ilev) = iop_field_v(model_start); + }); + } + if (model_end < total_nlevs) { + Kokkos::parallel_for(Kokkos::RangePolicy<>(model_end, total_nlevs), + KOKKOS_LAMBDA (const int ilev) { + iop_field_v(ilev) = iop_field_v(model_end-1); + }); + } + } + } + } + + // Now that data is loaded, reset the index of the currently loaded data. + m_time_info.time_idx_of_current_data = iop_file_time_idx; +} + +void IntensiveObservationPeriod:: +set_fields_from_iop_data(const field_mgr_ptr field_mgr) +{ + if (m_params.get("zero_non_iop_tracers") && field_mgr->has_group("tracers")) { + // Zero out all tracers before setting iop tracers (if requested) + field_mgr->get_field_group("tracers").m_bundle->deep_copy(0); + } + + EKAT_REQUIRE_MSG(field_mgr->get_grid()->name() == "Physics GLL", + "Error! Attempting to set non-GLL fields using " + "data from the IOP file.\n"); + + // Find which fields need to be written + const bool set_ps = field_mgr->has_field("ps") && has_iop_field("Ps"); + const bool set_T_mid = field_mgr->has_field("T_mid") && has_iop_field("T"); + const bool set_horiz_winds_u = field_mgr->has_field("horiz_winds") && has_iop_field("u"); + const bool set_horiz_winds_v = field_mgr->has_field("horiz_winds") && has_iop_field("v"); + const bool set_qv = field_mgr->has_field("qv") && has_iop_field("q"); + const bool set_nc = field_mgr->has_field("nc") && has_iop_field("NUMLIQ"); + const bool set_qc = field_mgr->has_field("qc") && has_iop_field("CLDLIQ"); + const bool set_qi = field_mgr->has_field("qi") && has_iop_field("CLDICE"); + const bool set_ni = field_mgr->has_field("ni") && has_iop_field("NUMICE"); + + // Create views/scalars for these field's data + view_1d ps; + view_2d T_mid, qv, nc, qc, qi, ni; + view_3d horiz_winds; + + Real ps_iop; + view_1d t_iop, u_iop, v_iop, qv_iop, nc_iop, qc_iop, qi_iop, ni_iop; + + if (set_ps) { + ps = field_mgr->get_field("ps").get_view(); + get_iop_field("Ps").sync_to_host(); + ps_iop = get_iop_field("Ps").get_view()(); + } + if (set_T_mid) { + T_mid = field_mgr->get_field("T_mid").get_view(); + t_iop = get_iop_field("T").get_view(); + } + if (set_horiz_winds_u || set_horiz_winds_v) { + horiz_winds = field_mgr->get_field("horiz_winds").get_view(); + if (set_horiz_winds_u) u_iop = get_iop_field("u").get_view(); + if (set_horiz_winds_v) v_iop = get_iop_field("v").get_view(); + } + if (set_qv) { + qv = field_mgr->get_field("qv").get_view(); + qv_iop = get_iop_field("q").get_view(); + } + if (set_nc) { + nc = field_mgr->get_field("nc").get_view(); + nc_iop = get_iop_field("NUMLIQ").get_view(); + } + if (set_qc) { + qc = field_mgr->get_field("qc").get_view(); + qc_iop = get_iop_field("CLDLIQ").get_view(); + } + if (set_qi) { + qi = field_mgr->get_field("qi").get_view(); + qi_iop = get_iop_field("CLDICE").get_view(); + } + if (set_ni) { + ni = field_mgr->get_field("ni").get_view(); + ni_iop = get_iop_field("NUMICE").get_view(); + } + + // Check if t_iop has any 0 entires near the top of the model + // and correct t_iop and q_iop accordingly. + correct_temperature_and_water_vapor(field_mgr); + + // Loop over all columns and copy IOP field values to FM views + const auto ncols = field_mgr->get_grid()->get_num_local_dofs(); + const auto nlevs = field_mgr->get_grid()->get_num_vertical_levels(); + const auto policy = ESU::get_default_team_policy(ncols, nlevs); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const KT::MemberType& team) { + const auto icol = team.league_rank(); + + if (set_ps) { + ps(icol) = ps_iop; + } + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlevs), [&] (const int ilev) { + if (set_T_mid) { + T_mid(icol, ilev) = t_iop(ilev); + } + if (set_horiz_winds_u) { + horiz_winds(icol, 0, ilev) = u_iop(ilev); + } + if (set_horiz_winds_v) { + horiz_winds(icol, 1, ilev) = v_iop(ilev); + } + if (set_qv) { + qv(icol, ilev) = qv_iop(ilev); + } + if (set_nc) { + nc(icol, ilev) = nc_iop(ilev); + } + if (set_qc) { + qc(icol, ilev) = qc_iop(ilev); + } + if (set_qi) { + qi(icol, ilev) = qi_iop(ilev); + } + if (set_ni) { + ni(icol, ilev) = ni_iop(ilev); + } + }); + }); +} + +void IntensiveObservationPeriod:: +correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr) +{ + // Find the first valid level index for t_iop, i.e., first non-zero entry + int first_valid_idx; + const auto nlevs = field_mgr->get_grid()->get_num_vertical_levels(); + auto t_iop = get_iop_field("T").get_view(); + Kokkos::parallel_reduce(nlevs, KOKKOS_LAMBDA (const int ilev, int& lmin) { + if (t_iop(ilev) > 0 && ilev < lmin) lmin = ilev; + }, Kokkos::Min(first_valid_idx)); + + // If first_valid_idx>0, we must correct IOP fields T and q corresponding to + // levels 0,...,first_valid_idx-1 + if (first_valid_idx > 0) { + // If we have values of T and q to correct, we must have both T_mid and qv as FM fields + EKAT_REQUIRE_MSG(field_mgr->has_field("T_mid"), "Error! IOP requires FM to define T_mid.\n"); + EKAT_REQUIRE_MSG(field_mgr->has_field("qv"), "Error! IOP requires FM to define qv.\n"); + + // Replace values of T and q where t_iop contains zeros + auto T_mid = field_mgr->get_field("T_mid").get_view(); + auto qv = field_mgr->get_field("qv").get_view(); + auto q_iop = get_iop_field("q").get_view(); + Kokkos::parallel_for(Kokkos::RangePolicy<>(0, first_valid_idx), KOKKOS_LAMBDA (const int ilev) { + t_iop(ilev) = T_mid(0, ilev); + q_iop(ilev) = qv(0, ilev); + }); + } +} + +} // namespace control +} // namespace scream + + diff --git a/components/eamxx/src/control/intensive_observation_period.hpp b/components/eamxx/src/control/intensive_observation_period.hpp new file mode 100644 index 000000000000..6e70f44a0f56 --- /dev/null +++ b/components/eamxx/src/control/intensive_observation_period.hpp @@ -0,0 +1,204 @@ +#ifndef SCREAM_IOP_HPP +#define SCREAM_IOP_HPP + +#include "share/scream_types.hpp" +#include "share/field/field_manager.hpp" +#include "share/grid/abstract_grid.hpp" +#include "share/util/scream_time_stamp.hpp" + +#include "ekat/ekat_parameter_list.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/mpi/ekat_comm.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" + +namespace scream { +namespace control { +/* + * Class which provides functionality for running EAMxx with an intensive + * observation period (IOP). Currently the only use case is the doubly + * periodic model (DP-SCREAM). + */ +class IntensiveObservationPeriod +{ + using vos = std::vector; + using field_mgr_ptr = std::shared_ptr; + using grid_ptr = std::shared_ptr; + + using KT = ekat::KokkosTypes; + using ESU = ekat::ExeSpaceUtils; + + template + using view_1d = KT::template view_1d; + template + using view_2d = KT::template view_2d; + template + using view_3d = KT::template view_3d; + template + using view_1d_host = typename view_1d::HostMirror; + using Pack1d = ekat::Pack; + +public: + + // Constructor + // Input: + // - comm: MPI communicator + // - params: Input yaml file needs intensive_observation_period_options sublist + // - run_t0: Initial timestamp for the simulation + // - model_nlevs: Number of vertical levels in the simulation. Needed since + // the iop file contains a (potentially) different number of levels + IntensiveObservationPeriod(const ekat::Comm& comm, + const ekat::ParameterList& params, + const util::TimeStamp& run_t0, + const int model_nlevs, + const Field& hyam, + const Field& hybm); + + // Default destructor + ~IntensiveObservationPeriod() = default; + + // Read data from IOP file and store internally. + void read_iop_file_data(const util::TimeStamp& current_ts); + + // Setup io grids for reading data from file and determine the closest lat/lon + // pair in a IC/topo file to the target lat/lon params for a specific grid. This + // should be called on each grid that loads field data from file before reading + // data since the data file is not guarenteed to contain lat/lon for the correct + // grid (e.g., loading PHIS_d from topography file which only contains lat/lon on + // PG2 grid). EAMxx expects the ic file to contain lat/lon on GLL grid, and + // topography file to contain lat/lon on PG2 grid. + void setup_io_info (const std::string& file_name, + const grid_ptr& grid); + + // Read ICs from file for IOP cases. We set all columns in the + // given fields to the values of the column in the file with the + // closest lat,lon pair to the target lat,lon in the parameters. + // The setup_io_info must be called for the correct grids before + // this function can be called. + // Input: + // - file_name: Name of the file used to load field data (IC or topo file) + // - field_names_nc: Field names used by the input file + // - field_names_eamxx: Field names used by eamxx + // - initial_ts: Inital timestamp + // Input/output + // - field_mgr: Field manager containing fields that need data read from files + void read_fields_from_file_for_iop(const std::string& file_name, + const vos& field_names_nc, + const vos& field_names_eamxx, + const util::TimeStamp& initial_ts, + const field_mgr_ptr field_mgr); + + // Version of above, but where nc and eamxx field names are identical + void read_fields_from_file_for_iop(const std::string& file_name, + const vos& field_names, + const util::TimeStamp& initial_ts, + const field_mgr_ptr field_mgr) + { + read_fields_from_file_for_iop(file_name, field_names, field_names, initial_ts, field_mgr); + } + + // Set fields using data loaded from the iop file + void set_fields_from_iop_data(const field_mgr_ptr field_mgr); + + // The IOP file may contain temperature values that are + // 0 at or above the surface. Correct these values using + // the temperature T_mid (from field_mgr) where we + // replace all values T_iop(k) == 0 with T_mid(0, k). + // Likewise, at these k indices, we will replace q_iop(k) + // with qv(0, k). + // Note: We only need to use the first column because during + // the loading of ICs, every columns will have the same + // data. + void correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr); + + // Store grid spacing for use in SHOC ad interface + void set_grid_spacing (const Real dx_short) { + m_dynamics_dx_size = dx_short*1000; + } + + Real get_dynamics_dx_size () { return m_dynamics_dx_size; } + + ekat::ParameterList& get_params() { return m_params; } + + bool has_iop_field(const std::string& fname) { + return m_iop_fields.count(fname) > 0; + } + + Field get_iop_field(const std::string& fname) { + EKAT_REQUIRE_MSG(has_iop_field(fname), "Error! Requesting IOP field \""+fname+"\", but field is not stored in object.\n"); + return m_iop_fields[fname]; + } + +private: + + // Struct for storing info related + // to the closest lat,lon pair + struct ClosestLatLonInfo { + // Value for the closest lat/lon in file. + Real closest_lat; + Real closest_lon; + // MPI rank which owns the columns whose + // lat,lon pair is closest to target lat, + // lon parameters. + int mpi_rank_of_closest_column; + // Local column index of closest lat,lon pair. + // Should be set -1 on ranks not equal to the + // one above. + int local_column_index_of_closest_column; + }; + + // Struct for storing relevant time information + struct TimeInfo { + util::TimeStamp iop_file_begin_time; + view_1d_host iop_file_times_in_sec; + + int time_idx_of_current_data = -1; + + int get_iop_file_time_idx (const util::TimeStamp& current_ts) + { + // Get iop file time index that the given timestamp falls between. + // Note: the last time in iop file represents the non-inclusive + // upper bound of acceptable model times. + const auto n_iop_times = iop_file_times_in_sec.extent(0); + int time_idx=-1; + for (size_t t=0; t=0, + "Error! Current model time ("+current_ts.to_string()+") is not within " + "IOP time period: ["+iop_file_begin_time.to_string()+", "+ + (iop_file_begin_time+iop_file_times_in_sec(n_iop_times-1)).to_string()+").\n"); + return time_idx; + } + }; + + void initialize_iop_file(const util::TimeStamp& run_t0, + int model_nlevs, + const Field& hyam, + const Field& hybm); + + ekat::Comm m_comm; + ekat::ParameterList m_params; + + std::map m_lat_lon_info; + TimeInfo m_time_info; + + Real m_dynamics_dx_size; + + std::map m_io_grids; + + std::map m_iop_fields; + std::map m_helper_fields; + + std::map m_iop_file_varnames; + std::map m_iop_field_surface_varnames; +}; // class IntensiveObservationPeriod + +} // namespace control +} // namespace scream + +#endif // #ifndef SCREAM_IOP_HPP + diff --git a/components/eamxx/src/control/tests/CMakeLists.txt b/components/eamxx/src/control/tests/CMakeLists.txt index 5bc8a490d93c..43aa5cebab1b 100644 --- a/components/eamxx/src/control/tests/CMakeLists.txt +++ b/components/eamxx/src/control/tests/CMakeLists.txt @@ -3,7 +3,9 @@ if (NOT ${SCREAM_BASELINES_ONLY}) include (ScreamUtils) # Unit test the ad - CreateUnitTest(ad_ut "ad_tests.cpp" "scream_control;scream_share" LABELS "driver") + CreateUnitTest(ad_ut "ad_tests.cpp" + LIBS scream_control + LABELS driver) # Copy yaml input file to run directory configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ad_tests.yaml diff --git a/components/eamxx/src/control/tests/ad_tests.cpp b/components/eamxx/src/control/tests/ad_tests.cpp index da6d3c335012..cd19e2a6fd76 100644 --- a/components/eamxx/src/control/tests/ad_tests.cpp +++ b/components/eamxx/src/control/tests/ad_tests.cpp @@ -16,7 +16,7 @@ TEST_CASE ("ad_tests","[!throws]") // Load ad parameter list std::string fname = "ad_tests.yaml"; ekat::ParameterList ad_params("Atmosphere Driver"); - REQUIRE_NOTHROW ( parse_yaml_file(fname,ad_params) ); + parse_yaml_file(fname,ad_params); // Create a comm ekat::Comm atm_comm (MPI_COMM_WORLD); diff --git a/components/eamxx/src/control/tests/ad_tests.yaml b/components/eamxx/src/control/tests/ad_tests.yaml index 8ac0a9bcb4cb..4b1c00865fc2 100644 --- a/components/eamxx/src/control/tests/ad_tests.yaml +++ b/components/eamxx/src/control/tests/ad_tests.yaml @@ -10,7 +10,7 @@ initial_conditions: Z: "A" atmosphere_processes: - atm_procs_list: (dummy1, dummy2, dummy3) + atm_procs_list: [dummy1, dummy2, dummy3] schedule_type: Sequential dummy1: @@ -28,6 +28,9 @@ atmosphere_processes: grids_manager: Type: Mesh Free - number_of_global_columns: 24 - number_of_vertical_levels: 3 + grids_names: ["Point Grid"] + Point Grid: + type: point_grid + number_of_global_columns: 24 + number_of_vertical_levels: 3 ... diff --git a/components/eamxx/src/control/tests/dummy_atm_proc.hpp b/components/eamxx/src/control/tests/dummy_atm_proc.hpp index d80d54ab36a2..ad11940c71c7 100644 --- a/components/eamxx/src/control/tests/dummy_atm_proc.hpp +++ b/components/eamxx/src/control/tests/dummy_atm_proc.hpp @@ -48,7 +48,8 @@ class DummyProcess : public scream::AtmosphereProcess { FieldLayout layout_vec ( {COL,CMP,LEV}, {num_cols,2,num_levs} ); if (m_dummy_type==A2G) { - add_field("A",layout,ekat::units::m,m_grid->name()); + // Check request by field/grid name only works + add_field("A",m_grid->name()); add_field("B",layout,ekat::units::m,m_grid->name(),"The Group"); add_field("C",layout,ekat::units::m,m_grid->name(),"The Group"); // These are not used at run time, but we use them to test diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index 7f0bd47e3296..5818c4d836a8 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -2,31 +2,23 @@ set(DIAGNOSTIC_SRCS atm_density.cpp dry_static_energy.cpp exner.cpp + field_at_height.cpp field_at_level.cpp field_at_pressure_level.cpp - ice_water_path.cpp - liquid_water_path.cpp longwave_cloud_forcing.cpp - meridional_vapor_flux.cpp potential_temperature.cpp - precip_ice_surf_mass_flux.cpp - precip_liq_surf_mass_flux.cpp - precip_total_surf_mass_flux.cpp - rain_water_path.cpp + precip_surf_mass_flux.cpp relative_humidity.cpp - rime_water_path.cpp sea_level_pressure.cpp shortwave_cloud_forcing.cpp - vapor_water_path.cpp - vertical_layer_interface.cpp - vertical_layer_midpoint.cpp - vertical_layer_thickness.cpp + surf_upward_latent_heat_flux.cpp + vapor_flux.cpp + vertical_layer.cpp virtual_temperature.cpp - zonal_vapor_flux.cpp + water_path.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) -target_include_directories(diagnostics PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../share) target_link_libraries(diagnostics PUBLIC scream_share) if (NOT SCREAM_LIB_ONLY) diff --git a/components/eamxx/src/diagnostics/atm_density.cpp b/components/eamxx/src/diagnostics/atm_density.cpp index 40fe23d73c07..24749ba1a1ef 100644 --- a/components/eamxx/src/diagnostics/atm_density.cpp +++ b/components/eamxx/src/diagnostics/atm_density.cpp @@ -1,16 +1,16 @@ #include "diagnostics/atm_density.hpp" +#include "share/util/scream_common_physics_functions.hpp" namespace scream { -// ========================================================================================= -AtmDensityDiagnostic::AtmDensityDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) +AtmDensityDiagnostic:: +AtmDensityDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) { // Nothing to do here } -// ========================================================================================= void AtmDensityDiagnostic::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -27,30 +27,32 @@ void AtmDensityDiagnostic::set_grids(const std::shared_ptr g // Set Field Layouts FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - constexpr int ps = Pack::n; // The fields required for this diagnostic to be computed - add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); + add_field("T_mid", scalar3d_layout_mid, K, grid_name, SCREAM_PACK_SIZE); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("qv", scalar3d_layout_mid, Q, grid_name, SCREAM_PACK_SIZE); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, kg/(m*m*m), grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(ps); + C_ap.request_allocation(SCREAM_PACK_SIZE); m_diagnostic_output.allocate_view(); } -// ========================================================================================= + void AtmDensityDiagnostic::compute_diagnostic_impl() { + using Pack = ekat::Pack; + using PF = scream::PhysicsFunctions; + const auto npacks = ekat::npack(m_num_levs); - const auto& atm_dens = m_diagnostic_output.get_view(); - const auto& T_mid = get_field_in("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& qv_mid = get_field_in("qv").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); + const auto atm_dens = m_diagnostic_output.get_view(); + const auto T_mid = get_field_in("T_mid").get_view(); + const auto p_mid = get_field_in("p_mid").get_view(); + const auto qv_mid = get_field_in("qv").get_view(); + const auto pseudo_density_mid = get_field_in("pseudo_density").get_view(); Kokkos::parallel_for("AtmosphereDensityDiagnostic", Kokkos::RangePolicy<>(0,m_num_cols*npacks), @@ -61,9 +63,6 @@ void AtmDensityDiagnostic::compute_diagnostic_impl() atm_dens(icol,jpack) = PF::calculate_density(pseudo_density_mid(icol,jpack),dz); }); Kokkos::fence(); - - const auto ts = get_field_in("T_mid").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } -// ========================================================================================= + } //namespace scream diff --git a/components/eamxx/src/diagnostics/atm_density.hpp b/components/eamxx/src/diagnostics/atm_density.hpp index d9ab9b6e65d7..0b62cdcd7e3b 100644 --- a/components/eamxx/src/diagnostics/atm_density.hpp +++ b/components/eamxx/src/diagnostics/atm_density.hpp @@ -2,27 +2,16 @@ #define EAMXX_ATM_DENSITY_DIAGNOSTIC_HPP #include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" namespace scream { -/* - * This diagnostic will produce the potential temperature. - */ - class AtmDensityDiagnostic : public AtmosphereDiagnostic { public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - // Constructors AtmDensityDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "AtmosphereDensity"; } diff --git a/components/eamxx/src/diagnostics/dry_static_energy.cpp b/components/eamxx/src/diagnostics/dry_static_energy.cpp index a9a66938fa1b..f4ed33ca56b6 100644 --- a/components/eamxx/src/diagnostics/dry_static_energy.cpp +++ b/components/eamxx/src/diagnostics/dry_static_energy.cpp @@ -1,11 +1,15 @@ #include "diagnostics/dry_static_energy.hpp" +#include "share/util/scream_common_physics_functions.hpp" + +#include namespace scream { // ========================================================================================= -DryStaticEnergyDiagnostic::DryStaticEnergyDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) +DryStaticEnergyDiagnostic:: +DryStaticEnergyDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) { // Nothing to do here } @@ -34,7 +38,7 @@ void DryStaticEnergyDiagnostic::set_grids(const std::shared_ptr("T_mid", scalar3d_layout_mid, K, grid_name, ps); add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); + add_field("qv", scalar3d_layout_mid, Q, grid_name, ps); add_field("phis", scalar2d_layout_col, m2/s2, grid_name, ps); // Construct and allocate the diagnostic field @@ -53,6 +57,8 @@ void DryStaticEnergyDiagnostic::set_grids(const std::shared_ptr; const auto npacks = ekat::npack(m_num_levs); const auto default_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(m_num_cols, npacks); @@ -70,8 +76,7 @@ void DryStaticEnergyDiagnostic::compute_diagnostic_impl() const int num_levs = m_num_levs; auto tmp_mid = m_tmp_mid; auto tmp_int = m_tmp_int; - Kokkos::deep_copy(tmp_mid,0.0); - Kokkos::deep_copy(tmp_int,0.0); + Kokkos::parallel_for("DryStaticEnergyDiagnostic", default_policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -83,10 +88,13 @@ void DryStaticEnergyDiagnostic::compute_diagnostic_impl() dz_s(jpack) = PF::calculate_dz(pseudo_density_mid(icol,jpack), p_mid(icol,jpack), T_mid(icol,jpack), qv_mid(icol,jpack)); }); team.team_barrier(); + PF::calculate_z_int(team,num_levs,dz_s,surf_geopotential,z_int_s); team.team_barrier(); + PF::calculate_z_mid(team,num_levs,z_int_s,z_mid_s); team.team_barrier(); + const auto& dse_s = ekat::subview(dse,icol); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, npacks), [&] (const Int& jpack) { dse_s(jpack) = PF::calculate_dse(T_mid(icol,jpack),z_mid_s(jpack),phis(icol)); @@ -94,9 +102,6 @@ void DryStaticEnergyDiagnostic::compute_diagnostic_impl() team.team_barrier(); }); Kokkos::fence(); - - const auto ts = get_field_in("T_mid").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } -// ========================================================================================= + } //namespace scream diff --git a/components/eamxx/src/diagnostics/dry_static_energy.hpp b/components/eamxx/src/diagnostics/dry_static_energy.hpp index b6348739c633..ab25e042add7 100644 --- a/components/eamxx/src/diagnostics/dry_static_energy.hpp +++ b/components/eamxx/src/diagnostics/dry_static_energy.hpp @@ -2,32 +2,23 @@ #define EAMXX_DRY_STATIC_ENERGY_DIAGNOSTIC_HPP #include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" + +#include "share/scream_types.hpp" +#include namespace scream { -/* - * This diagnostic will produce the potential temperature. - */ - class DryStaticEnergyDiagnostic : public AtmosphereDiagnostic { public: using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; using view_2d = typename KT::template view_2d; // Constructors DryStaticEnergyDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "DryStaticEnergy"; } diff --git a/components/eamxx/src/diagnostics/exner.cpp b/components/eamxx/src/diagnostics/exner.cpp index 72fe98009722..16f0d5da4491 100644 --- a/components/eamxx/src/diagnostics/exner.cpp +++ b/components/eamxx/src/diagnostics/exner.cpp @@ -1,11 +1,13 @@ #include "diagnostics/exner.hpp" +#include "share/util/scream_common_physics_functions.hpp" namespace scream { // ========================================================================================= -ExnerDiagnostic::ExnerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) +ExnerDiagnostic:: +ExnerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) { // Nothing to do here } @@ -24,21 +26,22 @@ void ExnerDiagnostic::set_grids(const std::shared_ptr grids_ m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - constexpr int ps = Pack::n; // The fields required for this diagnostic to be computed - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, nondim, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(ps); + C_ap.request_allocation(SCREAM_PACK_SIZE); m_diagnostic_output.allocate_view(); } // ========================================================================================= void ExnerDiagnostic::compute_diagnostic_impl() { + using Pack = ekat::Pack; + using PF = PhysicsFunctions; const auto npacks = ekat::npack(m_num_levs); const auto& exner = m_diagnostic_output.get_view(); @@ -52,9 +55,6 @@ void ExnerDiagnostic::compute_diagnostic_impl() exner(icol,jpack) = PF::exner_function(p_mid(icol,jpack)); }); Kokkos::fence(); - - const auto ts = get_field_in("p_mid").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } // ========================================================================================= } //namespace scream diff --git a/components/eamxx/src/diagnostics/exner.hpp b/components/eamxx/src/diagnostics/exner.hpp index 13150557cf8e..5e029b716bf0 100644 --- a/components/eamxx/src/diagnostics/exner.hpp +++ b/components/eamxx/src/diagnostics/exner.hpp @@ -2,7 +2,6 @@ #define EAMXX_EXNER_DIAGNOSTIC_HPP #include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" namespace scream { @@ -14,9 +13,6 @@ namespace scream class ExnerDiagnostic : public AtmosphereDiagnostic { public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - // Constructors ExnerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp new file mode 100644 index 000000000000..7759dfe5811d --- /dev/null +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -0,0 +1,187 @@ +#include "diagnostics/field_at_height.hpp" + +#include "ekat/std_meta/ekat_std_utils.hpp" +#include "ekat/util/ekat_units.hpp" + +namespace +{ +// Find first position in array pointed by [beg,end) that is below z +// If all z's in array are >=z, return end +template +KOKKOS_INLINE_FUNCTION +const T* find_first_smaller_z (const T* beg, const T* end, const T& z) +{ + // It's easier to find the last entry that is not smaller than z, + // and then we'll return the ptr after that + int count = end - beg; + while (count>1) { + auto mid = beg + count/2 - 1; + // if (z>=*mid) { + if (*mid>=z) { + beg = mid+1; + } else { + end = mid+1; + } + count = end - beg; + } + + return *beg < z ? beg : end; +} + +} // anonymous namespace + +namespace scream +{ + +FieldAtHeight:: +FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) +{ + m_field_name = m_params.get("field_name"); + const auto& location = m_params.get("vertical_location"); + auto chars_start = location.find_first_not_of("0123456789."); + EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, + "Error! Invalid string for height value for FieldAtHeight.\n" + " - input string : " + location + "\n" + " - expected format: Nm, with N integer\n"); + const auto z_str = location.substr(0,chars_start); + m_z = std::stod(z_str); + + const auto units = location.substr(chars_start); + EKAT_REQUIRE_MSG (units=="m", + "Error! Invalid string for height value for FieldAtHeight.\n" + " - input string : " + location + "\n" + " - expected format: Nm, with N integer\n"); + m_diag_name = m_field_name + "_at_" + m_params.get("vertical_location"); +} + +void FieldAtHeight:: +set_grids (const std::shared_ptr grids_manager) +{ + const auto& gname = m_params.get("grid_name"); + add_field(m_field_name,gname); + + // We don't know yet which one we need + add_field("z_mid",gname); + add_field("z_int",gname); +} + +void FieldAtHeight:: +initialize_impl (const RunType /*run_type*/) +{ + const auto& f = get_field_in(m_field_name); + const auto& fid = f.get_header().get_identifier(); + + // Sanity checks + using namespace ShortFieldTagsNames; + const auto& layout = fid.get_layout(); + EKAT_REQUIRE_MSG (f.data_type()==DataType::RealType, + "Error! FieldAtHeight only supports Real data type field.\n" + " - field name: " + fid.name() + "\n" + " - field data type: " + e2str(f.data_type()) + "\n"); + EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=3, + "Error! Field rank not supported by FieldAtHeight.\n" + " - field name: " + fid.name() + "\n" + " - field layout: " + to_string(layout) + "\n"); + const auto tag = layout.tags().back(); + EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, + "Error! FieldAtHeight diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" + " - field name : " + fid.name() + "\n" + " - field layout: " + to_string(layout) + "\n"); + + // Figure out the z value + m_z_name = tag==LEV ? "z_mid" : "z_int"; + + // All good, create the diag output + FieldIdentifier d_fid (m_diag_name,layout.strip_dim(tag),fid.get_units(),fid.get_grid_name()); + m_diagnostic_output = Field(d_fid); + m_diagnostic_output.allocate_view(); + + using stratts_t = std::map; + + // Propagate any io string attribute from input field to diag field + const auto& src = get_fields_in().front(); + const auto& src_atts = src.get_header().get_extra_data("io: string attributes"); + auto& dst_atts = m_diagnostic_output.get_header().get_extra_data("io: string attributes"); + for (const auto& [name, val] : src_atts) { + dst_atts[name] = val; + } +} + +// ========================================================================================= +void FieldAtHeight::compute_diagnostic_impl() +{ + const auto z_view = get_field_in(m_z_name).get_view(); + const Field& f = get_field_in(m_field_name); + const auto& fl = f.get_header().get_identifier().get_layout(); + + using RangePolicy = typename KokkosTypes::RangePolicy; + + auto z_tgt = m_z; + auto nlevs = fl.dims().back(); + if (fl.rank()==2) { + const auto f_view = f.get_view(); + const auto d_view = m_diagnostic_output.get_view(); + + RangePolicy policy (0,fl.dims()[0]); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const int i) { + auto f_i = ekat::subview(f_view,i); + auto z_i = ekat::subview(z_view,i); + + auto beg = z_i.data(); + auto end = beg+nlevs; + auto it = find_first_smaller_z(beg,end,z_tgt); + if (it==beg) { + // We just extapolate with first entry + d_view(i) = f_i(0); + } else if (it==end) { + // We just extapolate with last entry + d_view(i) = f_i(nlevs-1); + } else { + auto pos = it-beg; + auto z0 = z_i(pos-1); + auto z1 = z_i(pos); + auto f0 = f_i(pos-1); + auto f1 = f_i(pos); + + d_view(i) = ( (z_tgt-z0)*f1 + (z1-z_tgt)*f0 ) / (z1-z0); + } + }); + } else { + const auto f_view = f.get_view(); + const auto d_view = m_diagnostic_output.get_view(); + + const auto dim0 = fl.dims()[0]; + const auto dim1 = fl.dims()[1]; + RangePolicy policy (0,dim0*dim1); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const int idx) { + const int i = idx / dim1; + const int j = idx % dim1; + auto f_ij = ekat::subview(f_view,i,j); + auto z_i = ekat::subview(z_view,i); + + auto beg = z_i.data(); + auto end = beg+nlevs; + auto it = find_first_smaller_z(beg,end,z_tgt); + if (it==beg) { + // We just extapolate with first entry + d_view(i,j) = f_ij(0); + } else if (it==end) { + // We just extapolate with last entry + d_view(i,j) = f_ij(nlevs-1); + } else { + auto pos = it-beg; + auto z0 = z_i(pos-1); + auto z1 = z_i(pos); + auto f0 = f_ij(pos-1); + auto f1 = f_ij(pos); + + d_view(i,j) = ( (z_tgt-z0)*f1 + (z1-z_tgt)*f0 ) / (z1-z0); + } + }); + } +} + +} //namespace scream diff --git a/components/eamxx/src/diagnostics/field_at_height.hpp b/components/eamxx/src/diagnostics/field_at_height.hpp new file mode 100644 index 000000000000..54843b4f1a0e --- /dev/null +++ b/components/eamxx/src/diagnostics/field_at_height.hpp @@ -0,0 +1,43 @@ +#ifndef EAMXX_FIELD_AT_HEIGHT_HPP +#define EAMXX_FIELD_AT_HEIGHT_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream +{ + +/* + * This diagnostic will produce a slice of a field at a given height (above surface) + */ + +class FieldAtHeight : public AtmosphereDiagnostic +{ +public: + + // Constructors + FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The name of the diagnostic + std::string name () const { return m_diag_name; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + +protected: +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void compute_diagnostic_impl (); +protected: + void initialize_impl (const RunType /*run_type*/); + + std::string m_diag_name; + std::string m_z_name; + std::string m_field_name; + + Real m_z; +}; + +} //namespace scream + +#endif // EAMXX_FIELD_AT_HEIGHT_HPP diff --git a/components/eamxx/src/diagnostics/field_at_level.cpp b/components/eamxx/src/diagnostics/field_at_level.cpp index 63ccaa0723cc..3634cac2729c 100644 --- a/components/eamxx/src/diagnostics/field_at_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_level.cpp @@ -8,61 +8,88 @@ namespace scream // ========================================================================================= FieldAtLevel::FieldAtLevel (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereDiagnostic(comm,params) - , m_field_layout(m_params.get("Field Layout")) - , m_field_units(m_params.get("Field Units")) { - m_field_name = m_params.get("Field Name"); - - EKAT_REQUIRE_MSG (m_field_layout.rank()>1 && m_field_layout.rank()<=6, - "Error! Field rank not supported by FieldAtLevel.\n" - " - field name: " + m_field_name + "\n" - " - field layout: " + to_string(m_field_layout) + "\n"); - using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG (ekat::contains(std::vector{LEV,ILEV},m_field_layout.tags().back()), - "Error! FieldAtLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" - " - field name : " + m_field_name + "\n" - " - field layout: " + to_string(m_field_layout) + "\n"); + const auto& fname = m_params.get("field_name"); + const auto& location = m_params.get("vertical_location"); + m_diag_name = fname + "_at_" + location; } -// ========================================================================================= -void FieldAtLevel::set_grids(const std::shared_ptr /* grids_manager */) +void FieldAtLevel:: +set_grids (const std::shared_ptr grids_manager) { - const auto& gname = m_params.get("Grid Name"); + const auto& fname = m_params.get("field_name"); + const auto& gname = m_params.get("grid_name"); + add_field(fname,gname); +} - add_field(m_field_name, m_field_layout, m_field_units, gname); +void FieldAtLevel:: +initialize_impl (const RunType /*run_type*/) +{ + const auto& f = get_fields_in().front(); + // Sanity checks + using namespace ShortFieldTagsNames; + const auto& fid = f.get_header().get_identifier(); + const auto& layout = fid.get_layout(); + EKAT_REQUIRE_MSG (layout.rank()>1 && layout.rank()<=6, + "Error! Field rank not supported by FieldAtLevel.\n" + " - field name: " + fid.name() + "\n" + " - field layout: " + to_string(layout) + "\n"); + const auto tag = layout.tags().back(); + EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, + "Error! FieldAtLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" + " - field name : " + fid.name() + "\n" + " - field layout: " + to_string(layout) + "\n"); - // Calculate the actual level to slice field - const auto& lev_str = m_params.get("Field Level"); - if (lev_str=="tom") { + // Figure out the level + const auto& location = m_params.get("vertical_location"); + if (ekat::starts_with(location,"lev_")) { + const auto& lev = location.substr(4); + EKAT_REQUIRE_MSG (lev.find_first_not_of("0123456789")==std::string::npos, + "Error! Invalid level specification for FieldAtLevel diagnostic.\n" + " - input value: '" + location + "'\n" + " - expected: 'lev_N', with N an integer.\n"); + m_field_level = std::stoi(lev); + EKAT_REQUIRE_MSG (m_field_level bool { - return s.find_first_not_of("0123456789")==std::string::npos; - }; - EKAT_REQUIRE_MSG (is_int(lev_str), - "Error! Entry 'Field Level' must be 'tom', 'bot', or a string representation of an integer.\n"); - - m_field_level = std::stoi(lev_str); + EKAT_ERROR_MSG ( + "Error! Invalid level specification for FieldAtLevel diagnostic.\n" + " - input value: '" + location + "'\n" + " - expected: 'model_top','model_bot', or 'levN', with N an integer.\n"); } - EKAT_REQUIRE_MSG (m_field_level>=0 && m_field_level; + + // Propagate any io string attribute from input field to diag field + const auto& src = get_fields_in().front(); + const auto& src_atts = src.get_header().get_extra_data("io: string attributes"); + auto& dst_atts = m_diagnostic_output.get_header().get_extra_data("io: string attributes"); + for (const auto& [name, val] : src_atts) { + dst_atts[name] = val; + } } + // ========================================================================================= void FieldAtLevel::compute_diagnostic_impl() { - const auto& f = get_field_in(m_field_name); - const int dsize = m_field_layout.size () / m_field_layout.dims().back(); + const auto& f = get_fields_in().front(); + const auto& diag_layout = m_diagnostic_output.get_header().get_identifier().get_layout(); using RangePolicy = Kokkos::RangePolicy; - RangePolicy policy(0,dsize); + RangePolicy policy(0,diag_layout.size()); auto level = m_field_level; - switch (m_field_layout.rank()) { - case 2: + switch (diag_layout.rank()) { + case 1: { auto f_view = f.get_view(); auto d_view = m_diagnostic_output.get_view< Real*>(); @@ -71,11 +98,11 @@ void FieldAtLevel::compute_diagnostic_impl() }); } break; - case 3: + case 2: { auto f_view = f.get_view(); auto d_view = m_diagnostic_output.get_view< Real**>(); - const int dim1 = m_field_layout.dims()[1]; + const int dim1 = diag_layout.dims()[1]; Kokkos::parallel_for(m_diagnostic_output.name(),policy,KOKKOS_LAMBDA(const int idx) { const int i = idx / dim1; const int j = idx % dim1; @@ -83,12 +110,12 @@ void FieldAtLevel::compute_diagnostic_impl() }); } break; - case 4: + case 3: { auto f_view = f.get_view(); auto d_view = m_diagnostic_output.get_view< Real***>(); - const int dim1 = m_field_layout.dims()[1]; - const int dim2 = m_field_layout.dims()[2]; + const int dim1 = diag_layout.dims()[1]; + const int dim2 = diag_layout.dims()[2]; Kokkos::parallel_for(m_diagnostic_output.name(),policy,KOKKOS_LAMBDA(const int idx) { const int i = (idx / dim2) / dim1; const int j = (idx / dim2) % dim1; @@ -97,13 +124,13 @@ void FieldAtLevel::compute_diagnostic_impl() }); } break; - case 5: + case 4: { auto f_view = f.get_view(); auto d_view = m_diagnostic_output.get_view< Real****>(); - const int dim1 = m_field_layout.dims()[1]; - const int dim2 = m_field_layout.dims()[2]; - const int dim3 = m_field_layout.dims()[3]; + const int dim1 = diag_layout.dims()[1]; + const int dim2 = diag_layout.dims()[2]; + const int dim3 = diag_layout.dims()[3]; Kokkos::parallel_for(m_diagnostic_output.name(),policy,KOKKOS_LAMBDA(const int idx) { const int i = ((idx / dim3) / dim2) / dim1; const int j = ((idx / dim3) / dim2) % dim1; @@ -113,14 +140,14 @@ void FieldAtLevel::compute_diagnostic_impl() }); } break; - case 6: + case 5: { auto f_view = f.get_view(); auto d_view = m_diagnostic_output.get_view< Real*****>(); - const int dim1 = m_field_layout.dims()[1]; - const int dim2 = m_field_layout.dims()[2]; - const int dim3 = m_field_layout.dims()[3]; - const int dim4 = m_field_layout.dims()[4]; + const int dim1 = diag_layout.dims()[1]; + const int dim2 = diag_layout.dims()[2]; + const int dim3 = diag_layout.dims()[3]; + const int dim4 = diag_layout.dims()[4]; Kokkos::parallel_for(m_diagnostic_output.name(),policy,KOKKOS_LAMBDA(const int idx) { const int i = (((idx / dim4) / dim3) / dim2) / dim1; const int j = (((idx / dim4) / dim3) / dim2) % dim1; @@ -135,18 +162,4 @@ void FieldAtLevel::compute_diagnostic_impl() Kokkos::fence(); } -void FieldAtLevel::set_required_field_impl (const Field& f) -{ - const auto& f_fid = f.get_header().get_identifier(); - - // Now that we have the exact units of f, we can build the diagnostic field - FieldIdentifier d_fid (f_fid.name() + "@lev_" + std::to_string(m_field_level), - m_field_layout.strip_dim(m_field_layout.rank()-1), - f_fid.get_units(), - f_fid.get_grid_name()); - - m_diagnostic_output = Field(d_fid); - m_diagnostic_output.allocate_view(); -} - } //namespace scream diff --git a/components/eamxx/src/diagnostics/field_at_level.hpp b/components/eamxx/src/diagnostics/field_at_level.hpp index b49b332d7d49..b63cda0a1a38 100644 --- a/components/eamxx/src/diagnostics/field_at_level.hpp +++ b/components/eamxx/src/diagnostics/field_at_level.hpp @@ -9,7 +9,7 @@ namespace scream { /* - * This diagnostic will produce the potential temperature. + * This diagnostic will produce a slice of a field at a given vertical level index */ class FieldAtLevel : public AtmosphereDiagnostic @@ -21,7 +21,7 @@ class FieldAtLevel : public AtmosphereDiagnostic FieldAtLevel (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_field_name + "@lev_" + std::to_string(m_field_level); } + std::string name () const { return m_diag_name; } // Set the grid void set_grids (const std::shared_ptr grids_manager); @@ -32,15 +32,10 @@ class FieldAtLevel : public AtmosphereDiagnostic #endif void compute_diagnostic_impl (); protected: + void initialize_impl (const RunType /*run_type*/); - void set_required_field_impl (const Field& f); - - // Keep track of field dimensions - std::string m_field_name; - FieldLayout m_field_layout; - ekat::units::Units m_field_units; - - int m_field_level; + std::string m_diag_name; + int m_field_level; }; // class FieldAtLevel } //namespace scream diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp index 39c00055ae27..ea47b8c8f54e 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp @@ -1,4 +1,5 @@ #include "diagnostics/field_at_pressure_level.hpp" +#include "share/util/scream_vertical_interpolation.hpp" #include "ekat/std_meta/ekat_std_utils.hpp" #include "ekat/util/ekat_units.hpp" @@ -10,57 +11,91 @@ namespace scream FieldAtPressureLevel:: FieldAtPressureLevel (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereDiagnostic(comm,params) - , m_field_name(m_params.get("Field Name")) - , m_field_layout(m_params.get("Field Layout")) - , m_field_units(m_params.get("Field Units")) - , m_pressure_level(m_params.get("Field Target Pressure")) { - using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG (ekat::contains(std::vector{LEV,ILEV},m_field_layout.tags().back()), - "Error! FieldAtPressureLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" - " - field name : " + m_field_name + "\n" - " - field layout: " + to_string(m_field_layout) + "\n"); + m_field_name = m_params.get("field_name"); + + // Figure out the pressure value + const auto& location = m_params.get("vertical_location"); + auto chars_start = location.find_first_not_of("0123456789."); + EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, + "Error! Invalid string for pressure value for FieldAtPressureLevel.\n" + " - input string : " + location + "\n" + " - expected format: Nxyz, with N integer, and xyz='mb', 'hPa', or 'Pa'\n"); + const auto press_str = location.substr(0,chars_start); + m_pressure_level = std::stod(press_str); + + const auto units = location.substr(chars_start); + EKAT_REQUIRE_MSG (units=="mb" or units=="hPa" or units=="Pa", + "Error! Invalid string for pressure value for FieldAtPressureLevel.\n" + " - input string : " + location + "\n" + " - expected format: Nxyz, with N integer, and xyz='mb', 'hPa', or 'Pa'\n"); + + // Convert pressure level to Pa, the units of pressure in the simulation + if (units=="mb" || units=="hPa") { + m_pressure_level *= 100; + } - m_p_tgt = view_1d("",1); + m_p_tgt = view_1d("",1); Kokkos::deep_copy(m_p_tgt, m_pressure_level); m_mask_val = m_params.get("mask_value",Real(std::numeric_limits::max()/10.0)); + + m_diag_name = m_field_name + "_at_" + location; } -// ========================================================================================= void FieldAtPressureLevel:: -set_grids(const std::shared_ptr grids_manager) +set_grids (const std::shared_ptr grids_manager) { - using namespace ShortFieldTagsNames; - using namespace ekat::units; - const auto& gname = m_params.get("Grid Name"); - auto m_grid = grids_manager->get_grid(gname); - m_num_cols = m_grid->get_num_local_dofs(); + const auto& gname = m_params.get("grid_name"); + add_field(m_field_name,gname); + + // We don't know yet which one we need + add_field("p_mid",gname); + add_field("p_int",gname); +} - add_field(m_field_name, m_field_layout, m_field_units, gname); +void FieldAtPressureLevel:: +initialize_impl (const RunType /*run_type*/) +{ + const auto& f = get_field_in(m_field_name); + const auto& fid = f.get_header().get_identifier(); - m_pres_name = m_field_layout.tags().back()==LEV ? "p_mid" : "p_int"; - add_field(m_pres_name, m_field_layout, Pa, gname); - m_num_levs = m_field_layout.dims().back(); + // Sanity checks + using namespace ShortFieldTagsNames; + const auto& layout = fid.get_layout(); + EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=3, + "Error! Field rank not supported by FieldAtPressureLevel.\n" + " - field name: " + fid.name() + "\n" + " - field layout: " + to_string(layout) + "\n"); + const auto tag = layout.tags().back(); + EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, + "Error! FieldAtPressureLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" + " - field name : " + fid.name() + "\n" + " - field layout: " + to_string(layout) + "\n"); - // A field at a specific pressure level is just the same field w/ the LEV/ILEV dimension stripped from it. - FieldLayout diag_layout = m_field_layout.strip_dim(m_field_layout.tags().back()); - FieldIdentifier fid (name(),diag_layout, m_field_units, gname); - m_diagnostic_output = Field(fid); + // All good, create the diag output + FieldIdentifier d_fid (m_diag_name,layout.strip_dim(tag),fid.get_units(),fid.get_grid_name()); + m_diagnostic_output = Field(d_fid); m_diagnostic_output.allocate_view(); + m_pressure_name = tag==LEV ? "p_mid" : "p_int"; + m_num_levs = layout.dims().back(); + auto num_cols = layout.dims().front(); + // Take care of mask tracking for this field, in case it is needed. This has two steps: // 1. We need to actually track the masked columns, so we create a 2d (COL only) field. // NOTE: Here we assume that even a source field of rank 3+ will be masked the same // across all components so the mask is represented by a column-wise slice. - // 2. We also need to create a helper field that is used w/ the vertical remapper to set + // 2. We also need to create a helper field that may be used w/ the vertical remapper to set // the mask. This field is 3d (COLxLEV) to mimic the type of interpolation that is // being conducted on the source field. // Add a field representing the mask as extra data to the diagnostic field. - auto nondim = Units::nondimensional(); + auto nondim = ekat::units::Units::nondimensional(); + const auto& gname = fid.get_grid_name(); + std::string mask_name = name() + " mask"; - FieldLayout mask_layout( {COL}, {m_num_cols}); + FieldLayout mask_layout( {COL}, {num_cols}); FieldIdentifier mask_fid (mask_name,mask_layout, nondim, gname); Field diag_mask(mask_fid); diag_mask.allocate_view(); @@ -68,60 +103,64 @@ set_grids(const std::shared_ptr grids_manager) m_diagnostic_output.get_header().set_extra_data("mask_value",m_mask_val); // Allocate helper views - // Note that mPack is by design a pack of size 1, so we need to take into consideration the size of - // the source data which will be equal to the product of the number of source packs and the simulation - // pack size. - FieldLayout mask_src_layout( {COL, LEV}, {m_num_cols, m_num_levs}); + FieldLayout mask_src_layout( {COL, LEV}, {num_cols, m_num_levs}); FieldIdentifier mask_src_fid ("mask_tmp",mask_src_layout, nondim, gname); m_mask_field = Field(mask_src_fid); m_mask_field.allocate_view(); - + + using stratts_t = std::map; + + // Propagate any io string attribute from input field to diag field + const auto& src = get_fields_in().front(); + const auto& src_atts = src.get_header().get_extra_data("io: string attributes"); + auto& dst_atts = m_diagnostic_output.get_header().get_extra_data("io: string attributes"); + for (const auto& [name, val] : src_atts) { + dst_atts[name] = val; + } } + // ========================================================================================= void FieldAtPressureLevel::compute_diagnostic_impl() { using namespace scream::vinterp; //This is 2D source pressure - const Field& pressure_f = get_field_in(m_pres_name); - const auto pressure = pressure_f.get_view(); - auto pres = view_Nd("",pressure.extent_int(0),pressure.extent_int(1)); - Kokkos::deep_copy(pres,pressure); + const Field& pressure_f = get_field_in(m_pressure_name); + const auto pressure = pressure_f.get_view(); + view_Nd pres(pressure.data(),pressure.extent_int(0),pressure.extent_int(1)); + // Kokkos::deep_copy(pres,pressure); - //input field const Field& f = get_field_in(m_field_name); // The setup for interpolation varies depending on the rank of the input field: const int rank = f.rank(); m_mask_field.deep_copy(1.0); - auto mask_v_tmp = m_mask_field.get_view(); + auto mask_v_tmp = m_mask_field.get_view(); if (rank==2) { - const auto f_data_src = f.get_view(); + const auto f_data_src = f.get_view(); //output field on new grid - auto d_data_tgt = m_diagnostic_output.get_view(); - view_Nd data_tgt_tmp(d_data_tgt.data(),d_data_tgt.extent_int(0),1); // Note, vertical interp wants a 2D view, so we create a temporary one + auto d_data_tgt = m_diagnostic_output.get_view(); + view_Nd data_tgt_tmp(d_data_tgt.data(),d_data_tgt.extent_int(0),1); // Note, vertical interp wants a 2D view, so we create a temporary one perform_vertical_interpolation(pres,m_p_tgt,f_data_src,data_tgt_tmp,m_num_levs,1,m_mask_val); // Track mask - auto extra_data = m_diagnostic_output.get_header().get_extra_data().at("mask_data"); - auto d_mask = ekat::any_cast(extra_data); - auto d_mask_tgt = d_mask.get_view(); - view_Nd mask_tgt_tmp(d_mask_tgt.data(),d_mask_tgt.extent_int(0),1); + auto mask = m_diagnostic_output.get_header().get_extra_data("mask_data"); + auto d_mask_tgt = mask.get_view(); + view_Nd mask_tgt_tmp(d_mask_tgt.data(),d_mask_tgt.extent_int(0),1); perform_vertical_interpolation(pres,m_p_tgt,mask_v_tmp,mask_tgt_tmp,m_num_levs,1,0); } else if (rank==3) { - const auto f_data_src = f.get_view(); + const auto f_data_src = f.get_view(); //output field on new grid - auto d_data_tgt = m_diagnostic_output.get_view(); - view_Nd data_tgt_tmp(d_data_tgt.data(),d_data_tgt.extent_int(0),d_data_tgt.extent_int(1),1); + auto d_data_tgt = m_diagnostic_output.get_view(); + view_Nd data_tgt_tmp(d_data_tgt.data(),d_data_tgt.extent_int(0),d_data_tgt.extent_int(1),1); perform_vertical_interpolation(pres,m_p_tgt,f_data_src,data_tgt_tmp,m_num_levs,1,m_mask_val); // Track mask - auto extra_data = m_diagnostic_output.get_header().get_extra_data().at("mask_data"); - auto d_mask = ekat::any_cast(extra_data); - auto d_mask_tgt = d_mask.get_view(); - view_Nd mask_tgt_tmp(d_mask_tgt.data(),d_mask_tgt.extent_int(0),1); + auto mask = m_diagnostic_output.get_header().get_extra_data("mask_data"); + auto d_mask_tgt = mask.get_view(); + view_Nd mask_tgt_tmp(d_mask_tgt.data(),d_mask_tgt.extent_int(0),1); perform_vertical_interpolation(pres,m_p_tgt,mask_v_tmp,mask_tgt_tmp,m_num_levs,1,0); } else { EKAT_ERROR_MSG("Error! field at pressure level only supports fields ranks 2 and 3 \n"); diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.hpp b/components/eamxx/src/diagnostics/field_at_pressure_level.hpp index 09179f145d8d..bdad35321f99 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.hpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.hpp @@ -2,19 +2,19 @@ #define EAMXX_FIELD_AT_PRESSURE_LEVEL_HPP #include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_vertical_interpolation.hpp" + +#include namespace scream { /* - * This diagnostic will produce the potential temperature. + * This diagnostic will produce a slice of a field at a given pressure level */ class FieldAtPressureLevel : public AtmosphereDiagnostic { public: - using mPack = ekat::Pack; using KT = KokkosTypes; template @@ -26,7 +26,7 @@ class FieldAtPressureLevel : public AtmosphereDiagnostic FieldAtPressureLevel (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_field_name + " @ pressure " + std::to_string(m_pressure_level) + " hPa"; } + std::string name () const { return m_diag_name; } // Set the grid void set_grids (const std::shared_ptr grids_manager); @@ -37,20 +37,18 @@ class FieldAtPressureLevel : public AtmosphereDiagnostic #endif void compute_diagnostic_impl (); protected: + void initialize_impl (const RunType /*run_type*/); + using Pack1 = ekat::Pack; - - // Keep track of field dimensions + std::string m_pressure_name; std::string m_field_name; - FieldLayout m_field_layout; - ekat::units::Units m_field_units; - std::string m_pres_name; + std::string m_diag_name; - view_1d m_p_tgt; + view_1d m_p_tgt; Field m_mask_field; Real m_pressure_level; int m_num_levs; - int m_num_cols; Real m_mask_val; }; // class FieldAtPressureLevel diff --git a/components/eamxx/src/diagnostics/ice_water_path.cpp b/components/eamxx/src/diagnostics/ice_water_path.cpp deleted file mode 100644 index 9d6ff7d6bda8..000000000000 --- a/components/eamxx/src/diagnostics/ice_water_path.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "diagnostics/ice_water_path.hpp" - -namespace scream -{ - -// ========================================================================================= -IceWaterPathDiagnostic::IceWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void IceWaterPathDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qi", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - - // Construct and allocate the diagnostic field - const auto m2 = m*m; - FieldIdentifier fid (name(), scalar2d_layout_mid, kg/m2, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void IceWaterPathDiagnostic::compute_diagnostic_impl() -{ - - using PC = scream::physics::Constants; - constexpr Real gravit = PC::gravit; - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, npacks); - const auto& lwp = m_diagnostic_output.get_view(); - const auto& qi_mid = get_field_in("qi").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - - const auto num_levs = m_num_levs; - Kokkos::parallel_for("IceWaterPathDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - lsum += qi_mid(icol,jpack)[klev] * pseudo_density_mid(icol,jpack)[klev]/gravit; - },lwp(icol)); - team.team_barrier(); - }); - - const auto ts = get_field_in("qi").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/ice_water_path.hpp b/components/eamxx/src/diagnostics/ice_water_path.hpp deleted file mode 100644 index 590afde868dc..000000000000 --- a/components/eamxx/src/diagnostics/ice_water_path.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef EAMXX_ICE_WATER_PATH_DIAGNOSTIC_HPP -#define EAMXX_ICE_WATER_PATH_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the potential temperature. - */ - -class IceWaterPathDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - - // Constructors - IceWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "IceWaterPath"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - -}; // class IceWaterPathDiagnostic - -} //namespace scream - -#endif // EAMXX_ICE_WATER_PATH_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/liquid_water_path.cpp b/components/eamxx/src/diagnostics/liquid_water_path.cpp deleted file mode 100644 index e85f81079e3b..000000000000 --- a/components/eamxx/src/diagnostics/liquid_water_path.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "diagnostics/liquid_water_path.hpp" - -namespace scream -{ - -// ========================================================================================= -LiqWaterPathDiagnostic::LiqWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void LiqWaterPathDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qc", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - - // Construct and allocate the diagnostic field - const auto m2 = m*m; - FieldIdentifier fid (name(), scalar2d_layout_mid, kg/m2, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void LiqWaterPathDiagnostic::compute_diagnostic_impl() -{ - - using PC = scream::physics::Constants; - constexpr Real gravit = PC::gravit; - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, npacks); - const auto& lwp = m_diagnostic_output.get_view(); - const auto& qc_mid = get_field_in("qc").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - - const auto num_levs = m_num_levs; - Kokkos::parallel_for("LiqWaterPathDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - lsum += qc_mid(icol,jpack)[klev] * pseudo_density_mid(icol,jpack)[klev]/gravit; - },lwp(icol)); - team.team_barrier(); - }); - - const auto ts = get_field_in("qc").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/liquid_water_path.hpp b/components/eamxx/src/diagnostics/liquid_water_path.hpp deleted file mode 100644 index ed36047381bf..000000000000 --- a/components/eamxx/src/diagnostics/liquid_water_path.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef EAMXX_LIQUID_WATER_PATH_DIAGNOSTIC_HPP -#define EAMXX_LIQUID_WATER_PATH_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the potential temperature. - */ - -class LiqWaterPathDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - - // Constructors - LiqWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "LiquidWaterPath"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - -}; // class LiqWaterPathDiagnostic - -} //namespace scream - -#endif // EAMXX_LIQUID_WATER_PATH_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp b/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp index b833eee15413..23ef59891e9f 100644 --- a/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp +++ b/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp @@ -1,5 +1,7 @@ #include "diagnostics/longwave_cloud_forcing.hpp" +#include + namespace scream { @@ -25,11 +27,10 @@ void LongwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr("LW_flux_up", scalar3d_layout_mid, W/m2, grid_name, ps); - add_field("LW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name, ps); + add_field("LW_flux_up", scalar3d_layout_mid, W/m2, grid_name); + add_field("LW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar2d_layout_col, W/m2, grid_name); @@ -41,22 +42,23 @@ void LongwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr::get_default_team_policy(m_num_cols,1); + using KT = KokkosTypes; + using ESU = ekat::ExeSpaceUtils; + using MemberType = typename KT::MemberType; + + const auto default_policy = ESU::get_default_team_policy(m_num_cols,1); const auto& LWCF = m_diagnostic_output.get_view(); - const auto& LW_flux_up = get_field_in("LW_flux_up").get_view(); - const auto& LW_clrsky_flux_up = get_field_in("LW_clrsky_flux_up").get_view(); + const auto& LW_flux_up = get_field_in("LW_flux_up").get_view(); + const auto& LW_clrsky_flux_up = get_field_in("LW_clrsky_flux_up").get_view(); Kokkos::parallel_for("LongwaveCloudForcingDiagnostic", default_policy, KOKKOS_LAMBDA(const MemberType& team) { const int icol = team.league_rank(); - LWCF(icol) = LW_clrsky_flux_up(icol,0)[0] - LW_flux_up(icol,0)[0] ; + LWCF(icol) = LW_clrsky_flux_up(icol,0) - LW_flux_up(icol,0) ; }); Kokkos::fence(); - - const auto ts = get_field_in("LW_flux_up").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } -// ========================================================================================= + } //namespace scream diff --git a/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp b/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp index 49e16ea92ed3..9703330ce0cb 100644 --- a/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp +++ b/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp @@ -2,8 +2,6 @@ #define EAMXX_LONGWAVE_CLOUD_FORCING_DIAGNOSTIC_HPP #include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" namespace scream { @@ -15,13 +13,6 @@ namespace scream class LongwaveCloudForcingDiagnostic : public AtmosphereDiagnostic { public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - // Constructors LongwaveCloudForcingDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -29,7 +20,7 @@ class LongwaveCloudForcingDiagnostic : public AtmosphereDiagnostic AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } // The name of the diagnostic - std::string name () const { return "LongWaveCloudForcing"; } + std::string name () const { return "LongwaveCloudForcing"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/meridional_vapor_flux.cpp b/components/eamxx/src/diagnostics/meridional_vapor_flux.cpp deleted file mode 100644 index 73bc0e80a7ed..000000000000 --- a/components/eamxx/src/diagnostics/meridional_vapor_flux.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include "diagnostics/meridional_vapor_flux.hpp" - -namespace scream -{ - -// ========================================================================================= -MeridionalVapFluxDiagnostic::MeridionalVapFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void MeridionalVapFluxDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto vel = m/s; - vel.set_string("m/s"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - FieldLayout horiz_wind_layout { {COL,CMP,LEV}, {m_num_cols,2,m_num_levs} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - // Note both u and v are packaged into the single field horiz_winds. - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - - - // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d_layout_mid, kg/m/s, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void MeridionalVapFluxDiagnostic::compute_diagnostic_impl() -{ - - using PC = scream::physics::Constants; - constexpr Real gravit = PC::gravit; - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, npacks); - const auto& qv_vert_integrated_flux_v = m_diagnostic_output.get_view(); - const auto& qv_mid = get_field_in("qv").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - const auto& horiz_winds = get_field_in("horiz_winds").get_view(); - - const auto num_levs = m_num_levs; - Kokkos::parallel_for("MeridionalVapFluxDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - // Note, horiz_winds contains u (index 0) and v (index 1). Here we want v - lsum += horiz_winds(icol,1,jpack)[klev] * qv_mid(icol,jpack)[klev] * pseudo_density_mid(icol,jpack)[klev]/gravit; - },qv_vert_integrated_flux_v(icol)); - team.team_barrier(); - }); - - const auto ts = get_field_in("qv").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/meridional_vapor_flux.hpp b/components/eamxx/src/diagnostics/meridional_vapor_flux.hpp deleted file mode 100644 index fe5615d93b28..000000000000 --- a/components/eamxx/src/diagnostics/meridional_vapor_flux.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef EAMXX_MERIDIONAL_VAPOR_FLUX_DIAGNOSTIC_HPP -#define EAMXX_MERIDIONAL_VAPOR_FLUX_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the meridional water vapor flux. - */ - -class MeridionalVapFluxDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - - // Constructors - MeridionalVapFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "MeridionalVaporFlux"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - -}; // class MeridonalVapFluxDiagnostic - -} //namespace scream - -#endif // EAMXX_MERIDIONAL_VAPOR_FLUX_HPP diff --git a/components/eamxx/src/diagnostics/potential_temperature.cpp b/components/eamxx/src/diagnostics/potential_temperature.cpp index 191d05f1de9f..ff197eaedea1 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.cpp +++ b/components/eamxx/src/diagnostics/potential_temperature.cpp @@ -54,9 +54,6 @@ void PotentialTemperatureDiagnostic::compute_diagnostic_impl() theta(icol,jpack) = PF::calculate_theta_from_T(T_mid(icol,jpack),p_mid(icol,jpack)); }); Kokkos::fence(); - - const auto ts = get_field_in("T_mid").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } // ========================================================================================= diff --git a/components/eamxx/src/diagnostics/precip_ice_surf_mass_flux.cpp b/components/eamxx/src/diagnostics/precip_ice_surf_mass_flux.cpp deleted file mode 100644 index 37a88514945e..000000000000 --- a/components/eamxx/src/diagnostics/precip_ice_surf_mass_flux.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "diagnostics/precip_ice_surf_mass_flux.hpp" - -namespace scream -{ - -// ========================================================================================= -PrecipIceSurfMassFluxDiagnostic::PrecipIceSurfMassFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void PrecipIceSurfMassFluxDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - const auto m2 = m*m; - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - - // The fields required for this diagnostic to be computed - add_field("precip_ice_surf_mass", scalar2d_layout_mid, kg/m2, grid_name); - - // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d_layout_mid, m/s, grid_name); - m_diagnostic_output = Field(fid); - m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void PrecipIceSurfMassFluxDiagnostic::compute_diagnostic_impl() -{ - const auto& mass_field = get_field_in("precip_ice_surf_mass"); - const auto& mass_view = mass_field.get_view(); - const auto& flux_view = m_diagnostic_output.get_view(); - - const auto& mass_track = mass_field.get_header().get_tracking(); - - const auto& t_start = mass_track.get_accum_start_time (); - const auto& t_now = mass_track.get_time_stamp (); - const auto dt = t_now - t_start; - - auto rhodt = PC::RHO_H2O*dt; - - Kokkos::parallel_for("PrecipIceSurfMassFluxDiagnostic", - KT::RangePolicy(0,m_num_cols), - KOKKOS_LAMBDA(const Int& icol) { - flux_view(icol) = mass_view(icol) / rhodt; - }); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/precip_ice_surf_mass_flux.hpp b/components/eamxx/src/diagnostics/precip_ice_surf_mass_flux.hpp deleted file mode 100644 index ad324f86a83a..000000000000 --- a/components/eamxx/src/diagnostics/precip_ice_surf_mass_flux.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef EAMXX_PRECIP_ICE_SURF_MASS_DIAGNOSTIC_HPP -#define EAMXX_PRECIP_ICE_SURF_MASS_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "physics/share/physics_constants.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the precipitation (ice) surface mass flux. - */ - -class PrecipIceSurfMassFluxDiagnostic : public AtmosphereDiagnostic -{ -public: - using PC = scream::physics::Constants; - using KT = ekat::KokkosTypes; - - // Constructors - PrecipIceSurfMassFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "PrecipIceSurfMassFlux"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - Int m_num_cols; -}; // class PrecipIceSurfMassFluxDiagnostic - -} //namespace scream - -#endif // EAMXX_PRECIP_ICE_SURF_MASS_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/precip_liq_surf_mass_flux.cpp b/components/eamxx/src/diagnostics/precip_liq_surf_mass_flux.cpp deleted file mode 100644 index d38171f11cfb..000000000000 --- a/components/eamxx/src/diagnostics/precip_liq_surf_mass_flux.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "diagnostics/precip_liq_surf_mass_flux.hpp" - -namespace scream -{ - -// ========================================================================================= -PrecipLiqSurfMassFluxDiagnostic::PrecipLiqSurfMassFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void PrecipLiqSurfMassFluxDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - const auto m2 = m*m; - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - - // The fields required for this diagnostic to be computed - add_field("precip_liq_surf_mass", scalar2d_layout_mid, kg/m2, grid_name); - - // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d_layout_mid, m/s, grid_name); - m_diagnostic_output = Field(fid); - m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void PrecipLiqSurfMassFluxDiagnostic::compute_diagnostic_impl() -{ - const auto& mass_field = get_field_in("precip_liq_surf_mass"); - const auto& mass_view = mass_field.get_view(); - const auto& flux_view = m_diagnostic_output.get_view(); - - const auto& mass_track = mass_field.get_header().get_tracking(); - - const auto& t_start = mass_track.get_accum_start_time (); - const auto& t_now = mass_track.get_time_stamp (); - const auto dt = t_now - t_start; - - auto rhodt = PC::RHO_H2O*dt; - - Kokkos::parallel_for("PrecipLiqSurfMassFluxDiagnostic", - KT::RangePolicy(0,m_num_cols), - KOKKOS_LAMBDA(const Int& icol) { - flux_view(icol) = mass_view(icol) / rhodt; - }); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/precip_liq_surf_mass_flux.hpp b/components/eamxx/src/diagnostics/precip_liq_surf_mass_flux.hpp deleted file mode 100644 index e1044d87dcce..000000000000 --- a/components/eamxx/src/diagnostics/precip_liq_surf_mass_flux.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef EAMXX_PRECIP_LIQ_SURF_MASS_DIAGNOSTIC_HPP -#define EAMXX_PRECIP_LIQ_SURF_MASS_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "physics/share/physics_constants.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the precipitation (liquid) surface mass flux. - */ - -class PrecipLiqSurfMassFluxDiagnostic : public AtmosphereDiagnostic -{ -public: - using PC = scream::physics::Constants; - using KT = ekat::KokkosTypes; - - // Constructors - PrecipLiqSurfMassFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "PrecipLiqSurfMassFlux"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - Int m_num_cols; -}; // class PrecipLiqSurfMassFluxDiagnostic - -} //namespace scream - -#endif // EAMXX_PRECIP_LIQ_SURF_MASS_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp b/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp new file mode 100644 index 000000000000..068456f0522d --- /dev/null +++ b/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp @@ -0,0 +1,115 @@ +#include "diagnostics/precip_surf_mass_flux.hpp" + +#include "physics/share/physics_constants.hpp" + +namespace scream +{ + +// =============================================================================== +PrecipSurfMassFlux:: +PrecipSurfMassFlux (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) +{ + ci_string type = m_params.get("precip_type"); + if (type=="ice") { + m_type = s_ice; + } else if (type=="liq") { + m_type = s_liq; + } else if (type=="total") { + m_type = s_ice + s_liq; + } else { + EKAT_ERROR_MSG ("Error! Invalid choice for 'precip_type': " + type + "\n"); + } + m_name = "precip_" + type + "_surf_mass_flux"; +} + +// ============================================================================== +void PrecipSurfMassFlux:: +set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + const auto m2 = m*m; + + auto grid = grids_manager->get_grid("Physics"); + const auto& grid_name = grid->name(); + m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank + + FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; + + // The fields required for this diagnostic to be computed + if (m_type & s_ice) { + add_field("precip_ice_surf_mass", scalar2d_layout_mid, kg/m2, grid_name); + } + if (m_type & s_liq) { + add_field("precip_liq_surf_mass", scalar2d_layout_mid, kg/m2, grid_name); + } + + // Construct and allocate the diagnostic field + FieldIdentifier fid(name(), scalar2d_layout_mid, m/s, grid_name); + m_diagnostic_output = Field(fid); + m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); + m_diagnostic_output.allocate_view(); +} +// =============================================================================== +void PrecipSurfMassFlux::compute_diagnostic_impl() +{ + using KT = ekat::KokkosTypes; + using PC = scream::physics::Constants; + + Field::view_dev_t mass_liq_d, mass_ice_d; + + const auto use_liq = m_type & s_liq; + const auto use_ice = m_type & s_ice; + + std::int64_t dt; + if (use_ice) { + auto mass_ice = get_field_in("precip_ice_surf_mass"); + mass_ice_d = mass_ice.get_view(); + + const auto& t_start = mass_ice.get_header().get_tracking().get_accum_start_time (); + const auto& t_now = mass_ice.get_header().get_tracking().get_time_stamp (); + dt = t_now-t_start; + } + if (use_liq) { + auto mass_liq = get_field_in("precip_liq_surf_mass"); + mass_liq_d = mass_liq.get_view(); + + const auto& t_start = mass_liq.get_header().get_tracking().get_accum_start_time (); + const auto& t_now = mass_liq.get_header().get_tracking().get_time_stamp (); + if (use_ice) { + EKAT_REQUIRE_MSG (dt==(t_now-t_start), + "Error! Liquid and ice precip mass fields have different accumulation time stamps!\n"); + } else { + dt = t_now-t_start; + } + } + + if (dt==0) { + // We can't compute a flux if no time has passed. + // This may be happening b/c we're at t=0, and we have INSTANT output, + // which always outputs fields at t=0. To avoid FPE's, we simply return, + // setting the time stamp of the diag to invalid to signal that we did + // not successfully compute it. + m_diagnostic_output.get_header().get_tracking().invalidate_time_stamp(); + return; + } + + auto rhodt = PC::RHO_H2O*dt; + const auto& flux_view = m_diagnostic_output.get_view(); + Kokkos::parallel_for("PrecipSurfMassFlux", + KT::RangePolicy(0,m_num_cols), + KOKKOS_LAMBDA(const Int& icol) { + if (use_ice) { + flux_view(icol) = mass_ice_d(icol) / rhodt; + if (use_liq) { + flux_view(icol) += mass_liq_d(icol) / rhodt; + } + } else { + flux_view(icol) = mass_liq_d(icol) / rhodt; + } + }); +} + +} //namespace scream diff --git a/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp b/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp new file mode 100644 index 000000000000..68dd1251646e --- /dev/null +++ b/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp @@ -0,0 +1,43 @@ +#ifndef EAMXX_PRECIP_SURF_MASS_FLUX_HPP +#define EAMXX_PRECIP_SURF_MASS_FLUX_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream +{ + +/* + * This diagnostic will produce the precipitation (ice) surface mass flux. + */ + +class PrecipSurfMassFlux : public AtmosphereDiagnostic +{ +public: + // Constructors + PrecipSurfMassFlux (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The name of the diagnostic + std::string name () const { return m_name; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + +protected: +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void compute_diagnostic_impl (); +protected: + + Int m_num_cols; + + static constexpr int s_ice = 1; + static constexpr int s_liq = 2; + + int m_type; + std::string m_name; +}; + +} + +#endif // EAMXX_PRECIP_SURF_MASS_FLUX_HPP diff --git a/components/eamxx/src/diagnostics/precip_total_surf_mass_flux.cpp b/components/eamxx/src/diagnostics/precip_total_surf_mass_flux.cpp deleted file mode 100644 index c06964948bac..000000000000 --- a/components/eamxx/src/diagnostics/precip_total_surf_mass_flux.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include "diagnostics/precip_total_surf_mass_flux.hpp" - -namespace scream -{ - -// ========================================================================================= -PrecipTotalSurfMassFluxDiagnostic::PrecipTotalSurfMassFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void PrecipTotalSurfMassFluxDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - const auto m2 = m*m; - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - - // The fields required for this diagnostic to be computed - add_field("precip_liq_surf_mass", scalar2d_layout_mid, kg/m2, grid_name); - add_field("precip_ice_surf_mass", scalar2d_layout_mid, kg/m2, grid_name); - - // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d_layout_mid, m/s, grid_name); - m_diagnostic_output = Field(fid); - m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void PrecipTotalSurfMassFluxDiagnostic::compute_diagnostic_impl() -{ - const auto& ice_mass_field = get_field_in("precip_ice_surf_mass"); - const auto& ice_mass_view = ice_mass_field.get_view(); - const auto& ice_mass_track = ice_mass_field.get_header().get_tracking(); - const auto& ice_t_start = ice_mass_track.get_accum_start_time(); - const auto& ice_t_now = ice_mass_track.get_time_stamp(); - const auto ice_dt = ice_t_now - ice_t_start; - auto ice_rhodt = PC::RHO_H2O*ice_dt; - - const auto& liq_mass_field = get_field_in("precip_liq_surf_mass"); - const auto& liq_mass_view = liq_mass_field.get_view(); - const auto& liq_mass_track = liq_mass_field.get_header().get_tracking(); - const auto& liq_t_start = liq_mass_track.get_accum_start_time(); - const auto& liq_t_now = liq_mass_track.get_time_stamp(); - const auto liq_dt = liq_t_now - liq_t_start; - auto liq_rhodt = PC::RHO_H2O*liq_dt; - - const auto& flux_view = m_diagnostic_output.get_view(); - - Kokkos::parallel_for("PrecipTotalSurfMassFluxDiagnostic", - KT::RangePolicy(0,m_num_cols), - KOKKOS_LAMBDA(const Int& icol) { - flux_view(icol) = ice_mass_view(icol)/ice_rhodt + liq_mass_view(icol)/liq_rhodt; - }); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/precip_total_surf_mass_flux.hpp b/components/eamxx/src/diagnostics/precip_total_surf_mass_flux.hpp deleted file mode 100644 index 470758e727d2..000000000000 --- a/components/eamxx/src/diagnostics/precip_total_surf_mass_flux.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef EAMXX_PRECIP_TOTAL_SURF_MASS_DIAGNOSTIC_HPP -#define EAMXX_PRECIP_TOTAL_SURF_MASS_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "physics/share/physics_constants.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the total precipitation surface mass flux. - */ - -class PrecipTotalSurfMassFluxDiagnostic : public AtmosphereDiagnostic -{ -public: - using PC = scream::physics::Constants; - using KT = ekat::KokkosTypes; - - // Constructors - PrecipTotalSurfMassFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "PrecipTotalSurfMassFlux"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - Int m_num_cols; -}; // class PrecipTotalSurfMassFluxDiagnostic - -} //namespace scream - -#endif // EAMXX_PRECIP_TOTAL_SURF_MASS_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/rain_water_path.cpp b/components/eamxx/src/diagnostics/rain_water_path.cpp deleted file mode 100644 index fb5586f703e4..000000000000 --- a/components/eamxx/src/diagnostics/rain_water_path.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "diagnostics/rain_water_path.hpp" - -namespace scream -{ - -// ========================================================================================= -RainWaterPathDiagnostic::RainWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void RainWaterPathDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qr", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - - // Construct and allocate the diagnostic field - const auto m2 = m*m; - FieldIdentifier fid (name(), scalar2d_layout_mid, kg/m2, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void RainWaterPathDiagnostic::compute_diagnostic_impl() -{ - - using PC = scream::physics::Constants; - constexpr Real gravit = PC::gravit; - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, npacks); - const auto& lwp = m_diagnostic_output.get_view(); - const auto& qr_mid = get_field_in("qr").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - - const auto num_levs = m_num_levs; - Kokkos::parallel_for("RainWaterPathDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - lsum += qr_mid(icol,jpack)[klev] * pseudo_density_mid(icol,jpack)[klev]/gravit; - },lwp(icol)); - team.team_barrier(); - }); - - const auto ts = get_field_in("qr").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/rain_water_path.hpp b/components/eamxx/src/diagnostics/rain_water_path.hpp deleted file mode 100644 index 7fe48df8ab40..000000000000 --- a/components/eamxx/src/diagnostics/rain_water_path.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef EAMXX_RAIN_WATER_PATH_DIAGNOSTIC_HPP -#define EAMXX_RAIN_WATER_PATH_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the potential temperature. - */ - -class RainWaterPathDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - - // Constructors - RainWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "RainWaterPath"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - -}; // class RainWaterPathDiagnostic - -} //namespace scream - -#endif // EAMXX_RAIN_WATER_PATH_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 9e19f4c1a4c9..181a531a7c87 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -3,29 +3,22 @@ // Include all diagnostics #include "diagnostics/field_at_level.hpp" +#include "diagnostics/field_at_height.hpp" #include "diagnostics/potential_temperature.hpp" #include "diagnostics/atm_density.hpp" #include "diagnostics/exner.hpp" #include "diagnostics/virtual_temperature.hpp" -#include "diagnostics/vertical_layer_interface.hpp" -#include "diagnostics/vertical_layer_thickness.hpp" -#include "diagnostics/vertical_layer_midpoint.hpp" +#include "diagnostics/vertical_layer.hpp" #include "diagnostics/dry_static_energy.hpp" #include "diagnostics/sea_level_pressure.hpp" -#include "diagnostics/liquid_water_path.hpp" -#include "diagnostics/ice_water_path.hpp" -#include "diagnostics/rime_water_path.hpp" -#include "diagnostics/vapor_water_path.hpp" -#include "diagnostics/rain_water_path.hpp" +#include "diagnostics/water_path.hpp" #include "diagnostics/shortwave_cloud_forcing.hpp" #include "diagnostics/longwave_cloud_forcing.hpp" #include "diagnostics/relative_humidity.hpp" -#include "diagnostics/zonal_vapor_flux.hpp" -#include "diagnostics/meridional_vapor_flux.hpp" +#include "diagnostics/vapor_flux.hpp" #include "diagnostics/field_at_pressure_level.hpp" -#include "diagnostics/precip_liq_surf_mass_flux.hpp" -#include "diagnostics/precip_ice_surf_mass_flux.hpp" -#include "diagnostics/precip_total_surf_mass_flux.hpp" +#include "diagnostics/precip_surf_mass_flux.hpp" +#include "diagnostics/surf_upward_latent_heat_flux.hpp" namespace scream { @@ -33,28 +26,25 @@ inline void register_diagnostics () { auto& diag_factory = AtmosphereDiagnosticFactory::instance(); diag_factory.register_product("PotentialTemperature",&create_atmosphere_diagnostic); diag_factory.register_product("FieldAtLevel",&create_atmosphere_diagnostic); + diag_factory.register_product("FieldAtHeight",&create_atmosphere_diagnostic); diag_factory.register_product("FieldAtPressureLevel",&create_atmosphere_diagnostic); diag_factory.register_product("AtmosphereDensity",&create_atmosphere_diagnostic); diag_factory.register_product("Exner",&create_atmosphere_diagnostic); diag_factory.register_product("VirtualTemperature",&create_atmosphere_diagnostic); - diag_factory.register_product("VerticalLayerInterface",&create_atmosphere_diagnostic); - diag_factory.register_product("VerticalLayerThickness",&create_atmosphere_diagnostic); - diag_factory.register_product("VerticalLayerMidpoint",&create_atmosphere_diagnostic); + diag_factory.register_product("z_int",&create_atmosphere_diagnostic); + diag_factory.register_product("geopotential_int",&create_atmosphere_diagnostic); + diag_factory.register_product("z_mid",&create_atmosphere_diagnostic); + diag_factory.register_product("geopotential_mid",&create_atmosphere_diagnostic); + diag_factory.register_product("dz",&create_atmosphere_diagnostic); diag_factory.register_product("DryStaticEnergy",&create_atmosphere_diagnostic); diag_factory.register_product("SeaLevelPressure",&create_atmosphere_diagnostic); - diag_factory.register_product("LiqWaterPath",&create_atmosphere_diagnostic); - diag_factory.register_product("IceWaterPath",&create_atmosphere_diagnostic); - diag_factory.register_product("VapWaterPath",&create_atmosphere_diagnostic); - diag_factory.register_product("RainWaterPath",&create_atmosphere_diagnostic); - diag_factory.register_product("RimeWaterPath",&create_atmosphere_diagnostic); + diag_factory.register_product("WaterPath",&create_atmosphere_diagnostic); diag_factory.register_product("ShortwaveCloudForcing",&create_atmosphere_diagnostic); diag_factory.register_product("LongwaveCloudForcing",&create_atmosphere_diagnostic); diag_factory.register_product("RelativeHumidity",&create_atmosphere_diagnostic); - diag_factory.register_product("ZonalVapFlux",&create_atmosphere_diagnostic); - diag_factory.register_product("MeridionalVapFlux",&create_atmosphere_diagnostic); - diag_factory.register_product("PrecipLiqSurfMassFlux",&create_atmosphere_diagnostic); - diag_factory.register_product("PrecipIceSurfMassFlux",&create_atmosphere_diagnostic); - diag_factory.register_product("PrecipTotalSurfMassFlux",&create_atmosphere_diagnostic); + diag_factory.register_product("VaporFlux",&create_atmosphere_diagnostic); + diag_factory.register_product("precip_surf_mass_flux",&create_atmosphere_diagnostic); + diag_factory.register_product("surface_upward_latent_heat_flux",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/relative_humidity.cpp b/components/eamxx/src/diagnostics/relative_humidity.cpp index b6838ae1bc2f..e7ceca4b8951 100644 --- a/components/eamxx/src/diagnostics/relative_humidity.cpp +++ b/components/eamxx/src/diagnostics/relative_humidity.cpp @@ -5,22 +5,20 @@ namespace scream { - -// ========================================================================================= -RelativeHumidityDiagnostic::RelativeHumidityDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) +RelativeHumidityDiagnostic:: +RelativeHumidityDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) { // Nothing to do here } -// ========================================================================================= void RelativeHumidityDiagnostic::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; using namespace ShortFieldTagsNames; - - auto Q = kg/kg; + auto nondim = Units::nondimensional(); + auto Q = nondim; Q.set_string("kg/kg"); auto grid = grids_manager->get_grid("Physics"); @@ -29,28 +27,32 @@ void RelativeHumidityDiagnostic::set_grids(const std::shared_ptrget_num_vertical_levels(); // Number of levels per column FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - constexpr int ps = Pack::n; // The fields required for this diagnostic to be computed - add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - + add_field("T_mid", scalar3d_layout_mid, K, grid_name, SCREAM_PACK_SIZE); + add_field("p_dry_mid", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("qv", scalar3d_layout_mid, Q, grid_name, SCREAM_PACK_SIZE); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar3d_layout_mid, K, grid_name); + FieldIdentifier fid (name(), scalar3d_layout_mid, nondim, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(ps); + C_ap.request_allocation(SCREAM_PACK_SIZE); m_diagnostic_output.allocate_view(); } -// ========================================================================================= + void RelativeHumidityDiagnostic::compute_diagnostic_impl() { + using Pack = ekat::Pack; + const auto npacks = ekat::npack(m_num_levs); auto theta = m_diagnostic_output.get_view(); auto T_mid = get_field_in("T_mid").get_view(); - auto p_mid = get_field_in("p_mid").get_view(); + auto p_dry_mid = get_field_in("p_dry_mid").get_view(); + auto dp_wet = get_field_in("pseudo_density").get_view(); + auto dp_dry = get_field_in("pseudo_density_dry").get_view(); auto qv_mid = get_field_in("qv").get_view(); const auto& RH = m_diagnostic_output.get_view(); @@ -64,15 +66,12 @@ void RelativeHumidityDiagnostic::compute_diagnostic_impl() const int jpack = idx % npacks; const auto range_pack = ekat::range(jpack*Pack::n); const auto range_mask = range_pack < num_levs; - auto qv_sat_l = physics::qv_sat(T_mid(icol,jpack), p_mid(icol,jpack), false, range_mask, physics::MurphyKoop, "RelativeHumidityDiagnostic::compute_diagnostic_impl"); + auto qv_sat_l = physics::qv_sat_wet(T_mid(icol,jpack), p_dry_mid(icol,jpack), false, range_mask, dp_wet(icol,jpack), dp_dry(icol,jpack), + physics::MurphyKoop, "RelativeHumidityDiagnostic::compute_diagnostic_impl"); RH(icol,jpack) = qv_mid(icol,jpack)/qv_sat_l; }); Kokkos::fence(); - - const auto ts = get_field_in("qv").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } -// ========================================================================================= } //namespace scream diff --git a/components/eamxx/src/diagnostics/relative_humidity.hpp b/components/eamxx/src/diagnostics/relative_humidity.hpp index af0d5b248020..9d31f60500f3 100644 --- a/components/eamxx/src/diagnostics/relative_humidity.hpp +++ b/components/eamxx/src/diagnostics/relative_humidity.hpp @@ -2,35 +2,16 @@ #define EAMXX_RELATIVE_HUMIDITY_DIAGNOSTIC_HPP #include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" -#include "physics/share/physics_functions.hpp" -#include "physics/share/physics_saturation_impl.hpp" namespace scream { -/* - * This diagnostic will produce relative humidity. - */ - class RelativeHumidityDiagnostic : public AtmosphereDiagnostic { public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - - // Constructors RelativeHumidityDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "RelativeHumidity"; } diff --git a/components/eamxx/src/diagnostics/rime_water_path.cpp b/components/eamxx/src/diagnostics/rime_water_path.cpp deleted file mode 100644 index dfd13fddfaea..000000000000 --- a/components/eamxx/src/diagnostics/rime_water_path.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "diagnostics/rime_water_path.hpp" - -namespace scream -{ - -// ========================================================================================= -RimeWaterPathDiagnostic::RimeWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void RimeWaterPathDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qm", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - - // Construct and allocate the diagnostic field - const auto m2 = m*m; - FieldIdentifier fid (name(), scalar2d_layout_mid, kg/m2, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void RimeWaterPathDiagnostic::compute_diagnostic_impl() -{ - - using PC = scream::physics::Constants; - constexpr Real gravit = PC::gravit; - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, npacks); - const auto& mwp = m_diagnostic_output.get_view(); - const auto& qm_mid = get_field_in("qm").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - - const auto num_levs = m_num_levs; - Kokkos::parallel_for("RimeWaterPathDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - lsum += qm_mid(icol,jpack)[klev] * pseudo_density_mid(icol,jpack)[klev]/gravit; - },mwp(icol)); - team.team_barrier(); - }); - - const auto ts = get_field_in("qm").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/rime_water_path.hpp b/components/eamxx/src/diagnostics/rime_water_path.hpp deleted file mode 100644 index 30f7200b2f4a..000000000000 --- a/components/eamxx/src/diagnostics/rime_water_path.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef EAMXX_RIME_WATER_PATH_DIAGNOSTIC_HPP -#define EAMXX_RIME_WATER_PATH_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the rime water path. - */ - -class RimeWaterPathDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - - // Constructors - RimeWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "RimeWaterPath"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - -}; // class RimeWaterPathDiagnostic - -} //namespace scream - -#endif // EAMXX_Rime_WATER_PATH_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/sea_level_pressure.cpp b/components/eamxx/src/diagnostics/sea_level_pressure.cpp index 423d1593e327..54f6b52fbea8 100644 --- a/components/eamxx/src/diagnostics/sea_level_pressure.cpp +++ b/components/eamxx/src/diagnostics/sea_level_pressure.cpp @@ -4,8 +4,9 @@ namespace scream { // ========================================================================================= -SeaLevelPressureDiagnostic::SeaLevelPressureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) +SeaLevelPressureDiagnostic:: +SeaLevelPressureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) { // Nothing to do here } @@ -45,7 +46,6 @@ void SeaLevelPressureDiagnostic::set_grids(const std::shared_ptr(m_num_levs); const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols,1); @@ -63,9 +63,6 @@ void SeaLevelPressureDiagnostic::compute_diagnostic_impl() p_sealevel(icol) = PF::calculate_psl(T_mid(icol,pack_surf)[idx_surf],p_mid(icol,pack_surf)[idx_surf],phis(icol)); }); Kokkos::fence(); - - const auto ts = get_field_in("T_mid").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } // ========================================================================================= } //namespace scream diff --git a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp index 4df2673632b4..e0e815549c8a 100644 --- a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp +++ b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp @@ -1,16 +1,17 @@ #include "diagnostics/shortwave_cloud_forcing.hpp" +#include + namespace scream { -// ========================================================================================= -ShortwaveCloudForcingDiagnostic::ShortwaveCloudForcingDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) +ShortwaveCloudForcingDiagnostic:: +ShortwaveCloudForcingDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) { // Nothing to do here } -// ========================================================================================= void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -25,13 +26,12 @@ void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr("SW_flux_dn", scalar3d_layout_mid, W/m2, grid_name, ps); - add_field("SW_flux_up", scalar3d_layout_mid, W/m2, grid_name, ps); - add_field("SW_clrsky_flux_dn", scalar3d_layout_mid, W/m2, grid_name, ps); - add_field("SW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name, ps); + add_field("SW_flux_dn", scalar3d_layout_mid, W/m2, grid_name); + add_field("SW_flux_up", scalar3d_layout_mid, W/m2, grid_name); + add_field("SW_clrsky_flux_dn", scalar3d_layout_mid, W/m2, grid_name); + add_field("SW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar2d_layout_col, W/m2, grid_name); @@ -40,27 +40,28 @@ void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr::get_default_team_policy(m_num_cols,1); + using KT = KokkosTypes; + using ESU = ekat::ExeSpaceUtils; + using MemberType = typename KT::MemberType; + + const auto default_policy = ESU::get_default_team_policy(m_num_cols,1); const auto& SWCF = m_diagnostic_output.get_view(); - const auto& SW_flux_dn = get_field_in("SW_flux_dn").get_view(); - const auto& SW_flux_up = get_field_in("SW_flux_up").get_view(); - const auto& SW_clrsky_flux_dn = get_field_in("SW_clrsky_flux_dn").get_view(); - const auto& SW_clrsky_flux_up = get_field_in("SW_clrsky_flux_up").get_view(); + const auto& SW_flux_dn = get_field_in("SW_flux_dn").get_view(); + const auto& SW_flux_up = get_field_in("SW_flux_up").get_view(); + const auto& SW_clrsky_flux_dn = get_field_in("SW_clrsky_flux_dn").get_view(); + const auto& SW_clrsky_flux_up = get_field_in("SW_clrsky_flux_up").get_view(); Kokkos::parallel_for("ShortwaveCloudForcingDiagnostic", default_policy, KOKKOS_LAMBDA(const MemberType& team) { const int icol = team.league_rank(); - SWCF(icol) = (SW_flux_dn(icol,0)[0] - SW_flux_up(icol,0)[0]) - (SW_clrsky_flux_dn(icol,0)[0] - SW_clrsky_flux_up(icol,0)[0]); + SWCF(icol) = (SW_flux_dn(icol,0) - SW_flux_up(icol,0)) - (SW_clrsky_flux_dn(icol,0) - SW_clrsky_flux_up(icol,0)); }); Kokkos::fence(); - - const auto ts = get_field_in("SW_flux_dn").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } -// ========================================================================================= + } //namespace scream diff --git a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp index 6d53b36118e4..9d676338a765 100644 --- a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp +++ b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp @@ -2,8 +2,6 @@ #define EAMXX_SHORTWAVE_CLOUD_FORCING_DIAGNOSTIC_HPP #include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" namespace scream { @@ -15,13 +13,6 @@ namespace scream class ShortwaveCloudForcingDiagnostic : public AtmosphereDiagnostic { public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - // Constructors ShortwaveCloudForcingDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -29,7 +20,7 @@ class ShortwaveCloudForcingDiagnostic : public AtmosphereDiagnostic AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } // The name of the diagnostic - std::string name () const { return "ShortWaveCloudForcing"; } + std::string name () const { return "ShortwaveCloudForcing"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp new file mode 100644 index 000000000000..19a49ad595c5 --- /dev/null +++ b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp @@ -0,0 +1,65 @@ +#include "diagnostics/surf_upward_latent_heat_flux.hpp" + +#include "physics/share/physics_constants.hpp" + +namespace scream { + +// ============================================================================== +SurfaceUpwardLatentHeatFlux:: +SurfaceUpwardLatentHeatFlux(const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm, params) + , m_name("surface_upward_latent_heat_flux") + , cf_long_name("surface_upward_latent_heat_flux_due_to_evaporation") +{ + // In the future we may add options to include latent heat fluxes due to other water species. + // See precip_surf_mass_flux.hpp and *.cpp for an example. + // We'll need to change the cf_long_name, too, when this happens. +} + +// ============================================================================== +void SurfaceUpwardLatentHeatFlux:: +set_grids (const std::shared_ptr grids_manager) +{ + const auto m2 = ekat::units::m * ekat::units::m; + const auto W = ekat::units::W; + const auto surf_evap_units = ekat::units::kg / m2 / ekat::units::s; + + auto grid = grids_manager->get_grid("Physics"); + const auto& grid_name = grid->name(); + m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank + + FieldLayout scalar2d_layout_mid { {ShortFieldTagsNames::COL}, {m_num_cols} }; + + // The fields required for this diagnostic to be computed + // surf_evap is defined by SurfaceCouplingImporter + add_field("surf_evap", scalar2d_layout_mid, surf_evap_units, grid_name); + + // Construct and allocate the diagnostic field + FieldIdentifier fid(name(), scalar2d_layout_mid, W/m2, grid_name); + // handle parent class member variables + m_diagnostic_output = Field(fid); + m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); + m_diagnostic_output.allocate_view(); +} + +// ============================================================================== +void SurfaceUpwardLatentHeatFlux::compute_diagnostic_impl() +{ + using KT = ekat::KokkosTypes; + using PC = scream::physics::Constants; + + constexpr auto latent_heat_evap = PC::LatVap; // [J/kg] + + Field::view_dev_t evap_view_d; + + auto evap = get_field_in("surf_evap"); + evap_view_d = evap.get_view(); + const auto& flux_view = m_diagnostic_output.get_view(); + Kokkos::parallel_for("SurfaceUpwardLatentHeatFlux", + KT::RangePolicy(0, m_num_cols), + KOKKOS_LAMBDA (const Int& icol) { + flux_view(icol) = evap_view_d(icol) * latent_heat_evap; + }); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.hpp b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.hpp new file mode 100644 index 000000000000..935a30ea03c9 --- /dev/null +++ b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.hpp @@ -0,0 +1,42 @@ +#ifndef EAMXX_SURF_LATENT_HEAT_FLUX_HPP +#define EAMXX_SURF_LATENT_HEAT_FLUX_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream +{ + +/* + * This diagnostic will produce the surface latent heat flux. + */ + +class SurfaceUpwardLatentHeatFlux : public AtmosphereDiagnostic +{ +public: + // Constructors + SurfaceUpwardLatentHeatFlux (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The name of the diagnostic + // Future extensions to latent heat flux may include processes + // other than evaporation, so we don't inline the name function here. + std::string name () const { return m_name; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + +protected: +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void compute_diagnostic_impl (); +protected: + + Int m_num_cols; + + int m_type; + std::string m_name; + std::string cf_long_name; +}; + +} // namespace scream +#endif diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 177f587b7c4e..0882d3f6119d 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -1,50 +1,62 @@ # NOTE: tests inside this if statement won't be built in a baselines-only build + +function (createDiagTest test_name test_srcs) + CreateUnitTest(${test_name} "${test_srcs}" + LIBS diagnostics physics_share + LABELS diagnostics) +endfunction () + if (NOT SCREAM_BASELINES_ONLY) include(ScreamUtils) - set( NEED_LIBS scream_share diagnostics physics_share ) - # Test extracting a single level of a field - CreateUnitTest(field_at_level "field_at_level_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(field_at_level "field_at_level_tests.cpp") # Test interpolating a field onto a single pressure level - CreateUnitTest(field_at_pressure_level "field_at_pressure_level_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(field_at_pressure_level "field_at_pressure_level_tests.cpp") + # Test interpolating a field at a specific height + CreateDiagTest(field_at_height "field_at_height_tests.cpp") # Test potential temperature diagnostic - CreateUnitTest(potential_temperature "potential_temperature_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(potential_temperature "potential_temperature_test.cpp") + # Test exner diagnostic - CreateUnitTest(exner_function "exner_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(exner_function "exner_test.cpp") + # Test virtual temperature - CreateUnitTest(virtual_temperature "virtual_temperature_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(virtual_temperature "virtual_temperature_test.cpp") + # Test atmosphere density - CreateUnitTest(atmosphere_density "atm_density_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) - # Test vertical layer thickness (dz) - CreateUnitTest(vertical_layer_thickness "vertical_layer_thickness_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) - # Test vertical layer interface (z_int) - CreateUnitTest(vertical_layer_interface "vertical_layer_interface_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) - # Test vertical layer interface (z_mid) - CreateUnitTest(vertical_layer_midpoint "vertical_layer_midpoint_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(atmosphere_density "atm_density_test.cpp") + + # Test vertical layer (dz, z_int, z_mid) + CreateDiagTest(vertical_layer "vertical_layer_tests.cpp") + # Test dry static energy - CreateUnitTest(dry_static_energy "dry_static_energy_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(dry_static_energy "dry_static_energy_test.cpp") + # Test sea level pressure - CreateUnitTest(sea_level_pressure "sea_level_pressure_test.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(sea_level_pressure "sea_level_pressure_test.cpp") + # Test total water path - CreateUnitTest(water_path "water_path_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(water_path "water_path_tests.cpp") + # Test shortwave cloud forcing - CreateUnitTest(shortwave_cloud_forcing "shortwave_cloud_forcing_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(shortwave_cloud_forcing "shortwave_cloud_forcing_tests.cpp") + # Test longwave cloud forcing - CreateUnitTest(longwave_cloud_forcing "longwave_cloud_forcing_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(longwave_cloud_forcing "longwave_cloud_forcing_tests.cpp") + # Test Relative Humidity - CreateUnitTest(relative_humidity "relative_humidity_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) - # Test Zonal Vapor Flux - CreateUnitTest(zonal_vapor_flux "zonal_vapor_flux_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) - # Test Meridional Vapor Flux - CreateUnitTest(meridional_vapor_flux "meridional_vapor_flux_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) - # Test precipitation ice mass surface flux - CreateUnitTest(precip_ice_surf_mass_flux "precip_ice_surf_mass_flux_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) - # Test precipitation liq mass surface flux - CreateUnitTest(precip_liq_surf_mass_flux "precip_liq_surf_mass_flux_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) - # Test total precipitation mass surface flux - CreateUnitTest(precip_total_surf_mass_flux "precip_total_surf_mass_flux_tests.cpp" "${NEED_LIBS}" LABELS "diagnostics" ) + CreateDiagTest(relative_humidity "relative_humidity_tests.cpp") + + # Test Vapor Flux + CreateDiagTest(vapor_flux "vapor_flux_tests.cpp") + + # Test precipitation mass surface flux + CreateDiagTest(precip_surf_mass_flux "precip_surf_mass_flux_tests.cpp") + + # Test surface latent heat flux + CreateDiagTest(surface_upward_latent_heat_flux "surf_upward_latent_heat_flux_tests.cpp") endif() diff --git a/components/eamxx/src/diagnostics/tests/atm_density_test.cpp b/components/eamxx/src/diagnostics/tests/atm_density_test.cpp index cd56b2026ca3..6ef7e50abd2f 100644 --- a/components/eamxx/src/diagnostics/tests/atm_density_test.cpp +++ b/components/eamxx/src/diagnostics/tests/atm_density_test.cpp @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); diff --git a/components/eamxx/src/diagnostics/tests/dry_static_energy_test.cpp b/components/eamxx/src/diagnostics/tests/dry_static_energy_test.cpp index 2f7d91b4cabe..d7cf0f163ffc 100644 --- a/components/eamxx/src/diagnostics/tests/dry_static_energy_test.cpp +++ b/components/eamxx/src/diagnostics/tests/dry_static_energy_test.cpp @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); diff --git a/components/eamxx/src/diagnostics/tests/exner_test.cpp b/components/eamxx/src/diagnostics/tests/exner_test.cpp index fd90b862a85a..53c7c25e9230 100644 --- a/components/eamxx/src/diagnostics/tests/exner_test.cpp +++ b/components/eamxx/src/diagnostics/tests/exner_test.cpp @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); diff --git a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp new file mode 100644 index 000000000000..0d45eb62e879 --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp @@ -0,0 +1,226 @@ +#include "catch2/catch.hpp" + +#include "diagnostics/field_at_height.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/field/field_utils.hpp" +#include "share/util/scream_setup_random_test.hpp" + +namespace scream { + +TEST_CASE("field_at_height") +{ + using namespace ShortFieldTagsNames; + + register_diagnostics(); + + // Get an MPI comm group for test + ekat::Comm comm(MPI_COMM_WORLD); + + constexpr int nruns = 10; + + util::TimeStamp t0 ({2022,1,1},{0,0,0}); + + // Create a grids manager w/ a point grid + int ncols = 3; + int ndims = 4; + int nlevs = 10; + int num_global_cols = ncols*comm.size(); + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,num_global_cols); + gm->build_grids(); + auto grid = gm->get_grid("Point Grid"); + + // Create input test fields, as well as z_mid/int fields + const auto m = ekat::units::m; + FieldIdentifier s_mid_fid ("s_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); + FieldIdentifier s_int_fid ("s_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + FieldIdentifier v_mid_fid ("v_mid",FieldLayout({COL,CMP, LEV},{ncols,ndims,nlevs }),m,grid->name()); + FieldIdentifier v_int_fid ("v_int",FieldLayout({COL,CMP,ILEV},{ncols,ndims,nlevs+1}),m,grid->name()); + FieldIdentifier z_mid_fid ("z_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); + FieldIdentifier z_int_fid ("z_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + + Field s_mid (s_mid_fid); + Field s_int (s_int_fid); + Field v_mid (v_mid_fid); + Field v_int (v_int_fid); + Field z_mid (z_mid_fid); + Field z_int (z_int_fid); + + s_mid.allocate_view(); + s_int.allocate_view(); + v_mid.allocate_view(); + v_int.allocate_view(); + z_mid.allocate_view(); + z_int.allocate_view(); + + s_mid.get_header().get_tracking().update_time_stamp(t0); + s_int.get_header().get_tracking().update_time_stamp(t0); + v_mid.get_header().get_tracking().update_time_stamp(t0); + v_int.get_header().get_tracking().update_time_stamp(t0); + z_mid.get_header().get_tracking().update_time_stamp(t0); + z_int.get_header().get_tracking().update_time_stamp(t0); + + auto print = [&](const std::string& msg) { + if (comm.am_i_root()) { + std::cout << msg; + } + }; + + auto engine = scream::setup_random_test(&comm); + using IPDF = std::uniform_int_distribution; + + IPDF pdf_fields (0,1000); + IPDF pdf_levs (1,nlevs-1); + + // Lambda to create and run a diag, and return output + auto run_diag = [&](const Field& f, const Field& z, + const std::string& loc) { + util::TimeStamp t0 ({2022,1,1},{0,0,0}); + auto& factory = AtmosphereDiagnosticFactory::instance(); + ekat::ParameterList pl; + pl.set("vertical_location",loc); + pl.set("field_name",f.name()); + pl.set("grid_name",grid->name()); + auto diag = factory.create("FieldAtheight",comm,pl); + diag->set_grids(gm); + diag->set_required_field(f); + diag->set_required_field(z); + diag->initialize(t0,RunType::Initial); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + return diag->get_diagnostic(); + }; + + // Create z(i,j)=nlevs-j, which makes testing easier + for (auto f : {z_mid, z_int}) { + auto v = f.get_view(); + const auto& dims = f.get_header().get_identifier().get_layout().dims(); + for (int i=0; i(); + const auto& size = f.get_header().get_identifier().get_layout().size(); + for (int i=0; i Testing with z_tgt coinciding with a z level\n"); + { + print(" -> scalar midpoint field...............\n"); + auto d = run_diag (s_mid,z_mid,loc); + auto tgt = s_mid.subfield(1,static_cast(lev_tgt)); + REQUIRE (views_are_equal(d,tgt,&comm)); + print(" -> scalar midpoint field............... OK!\n"); + } + { + print(" -> scalar interface field...............\n"); + auto d = run_diag (s_int,z_int,loc); + // z_mid = nlevs+1-ilev, so the tgt slice is nlevs+1-z_tgt + auto tgt = s_int.subfield(1,static_cast(lev_tgt)); + REQUIRE (views_are_equal(d,tgt,&comm)); + print(" -> scalar interface field............... OK!\n"); + } + { + print(" -> vector midpoint field...............\n"); + auto d = run_diag (v_mid,z_mid,loc); + // We can't subview over 3rd index and keep layout right, + // so do all cols separately + for (int i=0; i(lev_tgt)); + REQUIRE (views_are_equal(di,tgt,&comm)); + } + print(" -> vector midpoint field............... OK!\n"); + } + { + print(" -> vector interface field...............\n"); + auto d = run_diag (v_int,z_int,loc); + // We can't subview over 3rd index and keep layout right, + // so do all cols separately + for (int i=0; i(lev_tgt)); + REQUIRE (views_are_equal(di,tgt,&comm)); + } + print(" -> vector interface field............... OK!\n"); + } + + z_tgt = pdf_levs(engine) + 0.5; + lev_tgt = nlevs-z_tgt; + loc = std::to_string(z_tgt) + "m"; + + auto zp1 = static_cast(std::round(lev_tgt+0.5)); + auto zm1 = static_cast(std::round(lev_tgt-0.5)); + + print(" -> Testing with z_tgt between levels\n"); + { + print(" -> scalar midpoint field...............\n"); + auto d = run_diag (s_mid,z_mid,loc); + auto tgt = s_mid.subfield(1,zp1).clone(); + tgt.update(s_mid.subfield(1,zm1),0.5,0.5); + REQUIRE (views_are_equal(d,tgt,&comm)); + print(" -> scalar midpoint field............... OK!\n"); + } + { + print(" -> scalar interface field...............\n"); + auto d = run_diag (s_int,z_int,loc); + auto tgt = s_int.subfield(1,zp1).clone(); + tgt.update(s_int.subfield(1,zm1),0.5,0.5); + REQUIRE (views_are_equal(d,tgt,&comm)); + print(" -> scalar interface field............... OK!\n"); + } + { + print(" -> vector midpoint field...............\n"); + auto d = run_diag (v_mid,z_mid,loc); + // We can't subview over 3rd index and keep layout right, + // so do all cols separately + for (int i=0; i vector midpoint field............... OK!\n"); + } + { + print(" -> vector interface field...............\n"); + auto d = run_diag (v_int,z_int,loc); + // We can't subview over 3rd index and keep layout right, + // so do all cols separately + for (int i=0; i vector interface field............... OK!\n"); + } + } +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp index 128bf1c9de2d..44b4cd851b9a 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp @@ -13,11 +13,7 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { const int num_global_cols = ncols*comm.size(); - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_vertical_levels", nlevs); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,num_global_cols); gm->build_grids(); return gm; @@ -71,35 +67,31 @@ TEST_CASE("field_at_level") auto f_mid_1 = f_mid.get_component(1); ekat::ParameterList params_mid, params_int; - params_mid.set("Field Name",f_mid_1.name()); - params_mid.set("Field Units",fid_mid.get_units()); - params_mid.set("Field Layout",fid_mid.get_layout().strip_dim(CMP)); - params_mid.set("Grid Name",fid_mid.get_grid_name()); - params_int.set("Field Name",f_int.name()); - params_int.set("Field Units",fid_int.get_units()); - params_int.set("Field Layout",fid_int.get_layout()); - params_int.set("Grid Name",fid_int.get_grid_name()); + params_mid.set("field_name",f_mid_1.name()); + params_int.set("field_name",f_int.name()); + params_mid.set("grid_name",grid->name()); + params_int.set("grid_name",grid->name()); using IPDF = std::uniform_int_distribution; IPDF ipdf (1,nlevs-2); - for (const std::string& lev_loc : {"tom", "bot", "rand"}) { + for (const std::string& lev_loc : {"model_top", "model_bot", "rand"}) { int lev; std::string lev_str; if (lev_loc=="bot") { lev = nlevs-1; - lev_str = "bot"; - } else if (lev_loc=="tom") { + lev_str = lev_loc; + } else if (lev_loc=="model_top") { lev = 0; - lev_str = "tom"; + lev_str = lev_loc; } else { lev = ipdf(engine); - lev_str = std::to_string(lev); + lev_str = "lev_" + std::to_string(lev); } printf (" -> testing extraction at level: %s\n",lev_str.c_str()); // Create and setup diagnostics - params_mid.set("Field Level",lev_str); - params_int.set("Field Level",lev_str); + params_mid.set("vertical_location",lev_str); + params_int.set("vertical_location",lev_str); auto diag_mid = std::make_shared(comm,params_mid); auto diag_int = std::make_shared(comm,params_int); diff --git a/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp index c820ab0ce7ae..4e0deab1dfca 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp @@ -18,11 +18,7 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { const int num_global_cols = ncols*comm.size(); - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_vertical_levels", nlevs); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,num_global_cols); gm->build_grids(); return gm; @@ -43,9 +39,6 @@ struct PressureBnds Real p_surf = 100000.0; // 1000mb }; -std::shared_ptr -get_test_gm(const ekat::Comm& io_comm, const int num_gcols, const int num_levs); - std::shared_ptr get_test_fm(std::shared_ptr grid); @@ -66,7 +59,7 @@ TEST_CASE("field_at_pressure_level_p2") // Create a grids manager w/ a point grid int ncols = 3; int nlevs = 10; - auto gm = get_test_gm(comm,ncols,nlevs); + auto gm = create_gm(comm,ncols,nlevs); // Create a field manager for testing auto grid = gm->get_grid("Point Grid"); @@ -111,8 +104,7 @@ TEST_CASE("field_at_pressure_level_p2") diag_f.sync_to_host(); auto test2_diag_v = diag_f.get_view(); // Check the mask field inside the diag_f - auto mask_tmp = diag_f.get_header().get_extra_data().at("mask_data"); - auto mask_f = ekat::any_cast(mask_tmp); + auto mask_f = diag_f.get_header().get_extra_data("mask_data"); mask_f.sync_to_host(); auto test2_mask_v = mask_f.get_view(); // @@ -133,12 +125,10 @@ TEST_CASE("field_at_pressure_level_p2") diag_f.sync_to_host(); auto test2_diag_v = diag_f.get_view(); // Check the mask field inside the diag_f - auto mask_tmp = diag_f.get_header().get_extra_data().at("mask_data"); - auto mask_f = ekat::any_cast(mask_tmp); + auto mask_f = diag_f.get_header().get_extra_data("mask_data"); mask_f.sync_to_host(); auto test2_mask_v = mask_f.get_view(); - auto mask_val_tmp = diag_f.get_header().get_extra_data().at("mask_value"); - Real mask_val = ekat::any_cast(mask_val_tmp); + auto mask_val = diag_f.get_header().get_extra_data("mask_value"); // for (int icol=0;icol get_test_gm(const ekat::Comm& io_comm, const int num_gcols, const int num_levs) -{ - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns",num_gcols); - gm_params.set("number_of_vertical_levels",num_levs); - auto gm = create_mesh_free_grids_manager(io_comm,gm_params); - gm->build_grids(); - return gm; -} -/*===================================================================================================*/ std::shared_ptr get_test_fm(std::shared_ptr grid) { using namespace ekat::units; @@ -242,14 +222,11 @@ get_test_diag(const ekat::Comm& comm, std::shared_ptr fm, st auto field = fm->get_field(fname); auto fid = field.get_header().get_identifier(); ekat::ParameterList params; - params.set("Field Name",fname); - params.set("Field Units",fid.get_units()); - params.set("Field Layout",fid.get_layout()); - params.set("Grid Name",fid.get_grid_name()); - params.set("Field Target Pressure",plevel); + params.set("field_name",field.name()); + params.set("grid_name",fm->get_grid()->name()); + params.set("vertical_location",std::to_string(plevel) + "Pa"); auto diag = std::make_shared(comm,params); diag->set_grids(gm); - diag->set_required_field(field); for (const auto& req : diag->get_required_field_requests()) { auto req_field = fm->get_field(req.fid); diag->set_required_field(req_field); diff --git a/components/eamxx/src/diagnostics/tests/longwave_cloud_forcing_tests.cpp b/components/eamxx/src/diagnostics/tests/longwave_cloud_forcing_tests.cpp index 56a6435aee98..9469ac27f61c 100644 --- a/components/eamxx/src/diagnostics/tests/longwave_cloud_forcing_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/longwave_cloud_forcing_tests.cpp @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); diff --git a/components/eamxx/src/diagnostics/tests/meridional_vapor_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/meridional_vapor_flux_tests.cpp deleted file mode 100644 index cc1db3d1a193..000000000000 --- a/components/eamxx/src/diagnostics/tests/meridional_vapor_flux_tests.cpp +++ /dev/null @@ -1,186 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/meridional_vapor_flux.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/share/physics_constants.hpp" - -#include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "share/field/field_utils.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" - -#include - -namespace scream { - -std::shared_ptr -create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - - const int num_local_elems = 4; - const int np = 4; - const int num_global_cols = ncols*comm.size(); - - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); - gm->build_grids(); - - return gm; -} - -//-----------------------------------------------------------------------------------------------// -template -void run(std::mt19937_64& engine) -{ - using PC = scream::physics::Constants; - using Pack = ekat::Pack; - using KT = ekat::KokkosTypes; - using ExecSpace = typename KT::ExeSpace; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - using rview_1d = typename KT::template view_1d; - - const int packsize = SCREAM_PACK_SIZE; - constexpr int num_levs = packsize*2 + 1; // Number of levels to use for tests, make sure the last pack can also have some empty slots (packsize>1). - const int num_mid_packs = ekat::npack(num_levs); - - // A world comm - ekat::Comm comm(MPI_COMM_WORLD); - - // Create a grids manager - single column for these tests - const int ncols = 1; //TODO should be set to the size of the communication group. - auto gm = create_gm(comm,ncols,num_levs); - - // Kokkos Policy - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, num_mid_packs); - - // Input (randomized) views - view_1d qv("qv",num_mid_packs), - pseudo_density("pseudo_density",num_mid_packs), - v("v",num_mid_packs); - - - auto dview_as_real = [&] (const view_1d& vo) -> rview_1d { - return rview_1d(reinterpret_cast(vo.data()),vo.size()*packsize); - }; - - // Construct random input data - using RPDF = std::uniform_real_distribution; - RPDF pdf_qv(0.0,1e-3), - pdf_v(0.0,200), - pdf_pseudo_density(1.0,100.0); - - // A time stamp - util::TimeStamp t0 ({2022,1,1},{0,0,0}); - - // Construct the Diagnostic - ekat::ParameterList params; - register_diagnostics(); - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - auto diag = diag_factory.create("MeridionalVapFlux",comm,params); - diag->set_grids(gm); - - - // Set the required fields for the diagnostic. - std::map input_fields; - for (const auto& req : diag->get_required_field_requests()) { - Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(packsize); - f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - diag->set_required_field(f.get_const()); - input_fields.emplace(name,f); - } - - // Initialize the diagnostic - diag->initialize(t0,RunType::Initial); - - // Run tests - { - // Construct random data to use for test - // Get views of input data and set to random values - const auto& qv_f = input_fields["qv"]; - const auto& qv_v = qv_f.get_view(); - const auto& pseudo_density_f = input_fields["pseudo_density"]; - const auto& pseudo_density_v = pseudo_density_f.get_view(); - const auto& horiz_winds_f = input_fields["horiz_winds"]; - const auto& horiz_winds_v = horiz_winds_f.get_view(); - - - for (int icol=0;icol::quiet_NaN()); - Kokkos::deep_copy(v_sub,v); - - } - // ekat::genRandArray(phis_v, engine, pdf_surface); - - // Run diagnostic and compare with manual calculation - diag->compute_diagnostic(); - const auto& diag_out = diag->get_diagnostic(); - Field qv_vert_integrated_flux_v_f = diag_out.clone(); - qv_vert_integrated_flux_v_f.deep_copy(0.0); - qv_vert_integrated_flux_v_f.sync_to_dev(); - const auto& qv_vert_integrated_flux_v_v = qv_vert_integrated_flux_v_f.get_view(); - constexpr Real gravit = PC::gravit; - Kokkos::parallel_for("", policy, KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - lsum += horiz_winds_v(icol,1,jpack)[klev] * qv_v(icol,jpack)[klev] * pseudo_density_v(icol,jpack)[klev]/gravit; - },qv_vert_integrated_flux_v_v(icol)); - - }); - Kokkos::fence(); - REQUIRE(views_are_equal(diag_out,qv_vert_integrated_flux_v_f)); - } - - // Finalize the diagnostic - diag->finalize(); - -} // run() - -TEST_CASE("meridional_vapor_flux_test", "meridional_vapor_flux_test]"){ - // Run tests for both Real and Pack, and for (potentially) different pack sizes - using scream::Real; - using Device = scream::DefaultDevice; - - constexpr int num_runs = 5; - - auto engine = scream::setup_random_test(); - - printf(" -> Number of randomized runs: %d\n\n", num_runs); - - printf(" -> Testing Pack scalar type...",SCREAM_PACK_SIZE); - for (int irun=0; irun(engine); - } - printf("ok!\n"); - - printf("\n"); - -} // TEST_CASE - -} // namespace diff --git a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp index d5c17602b7d4..b11d8aa72ef6 100644 --- a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp +++ b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); diff --git a/components/eamxx/src/diagnostics/tests/precip_ice_surf_mass_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/precip_ice_surf_mass_flux_tests.cpp deleted file mode 100644 index 635ef48ccf83..000000000000 --- a/components/eamxx/src/diagnostics/tests/precip_ice_surf_mass_flux_tests.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/precip_ice_surf_mass_flux.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/share/physics_constants.hpp" - -#include "share/util/scream_setup_random_test.hpp" -#include "share/field/field_utils.hpp" - -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" - -#include - -namespace scream { - -std::shared_ptr -create_gm (const ekat::Comm& comm, const int ncols) { - - const int num_global_cols = ncols*comm.size(); - - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_vertical_levels", 1); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); - gm->build_grids(); - - return gm; -} - -//-----------------------------------------------------------------------------------------------// -template -void run(std::mt19937_64& engine) -{ - using PC = scream::physics::Constants; - using KT = ekat::KokkosTypes; - - // A world comm - ekat::Comm comm(MPI_COMM_WORLD); - - // Create a grids manager - const int ncols = 13; - auto gm = create_gm(comm,ncols); - - // Create timestep - const int dt=1800; - - // Construct random input data - using RPDF = std::uniform_real_distribution; - RPDF pdf_mass(0.0,1.0); - - // Initial time stamp - util::TimeStamp t0 ({2022,1,1},{0,0,0}); - - // Construct the Diagnostic - ekat::ParameterList params; - register_diagnostics(); - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - auto diag = diag_factory.create("PrecipIceSurfMassFlux",comm,params); - diag->set_grids(gm); - - // Set the required fields for the diagnostic. - std::map input_fields; - for (const auto& req : diag->get_required_field_requests()) { - Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(); - f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - f.get_header().get_tracking().set_accum_start_time(t0); - diag->set_required_field(f.get_const()); - REQUIRE_THROWS(diag->set_computed_field(f)); - input_fields.emplace(name,f); - } - - // Initialize the diagnostic - diag->initialize(t0,RunType::Initial); - - // Run tests - { - util::TimeStamp t = t0 + dt; - - // Construct random data to use for test - // Get views of input data and set to random values - auto precip_ice_surf_mass_f = input_fields["precip_ice_surf_mass"]; - const auto& precip_ice_surf_mass_v = precip_ice_surf_mass_f.get_view(); - for (int icol=0;icolcompute_diagnostic(); - const auto& diag_out = diag->get_diagnostic(); - Field theta_f = diag_out.clone(); - theta_f.deep_copy(0.0); - theta_f.sync_to_dev(); - const auto& theta_v = theta_f.get_view(); - const auto rhodt = PC::RHO_H2O*dt; - Kokkos::parallel_for("precip_ice_surf_mass_flux_test", - typename KT::RangePolicy(0,ncols), - KOKKOS_LAMBDA(const int& icol) { - theta_v(icol) = precip_ice_surf_mass_v(icol) / rhodt; - }); - Kokkos::fence(); - REQUIRE(views_are_equal(diag_out,theta_f)); - } - - // Finalize the diagnostic - diag->finalize(); - -} // run() - -TEST_CASE("precip_ice_surf_mass_flux_test", "precip_ice_surf_mass_flux_test]"){ - using scream::Real; - using Device = scream::DefaultDevice; - - constexpr int num_runs = 5; - - auto engine = scream::setup_random_test(); - - printf(" -> Number of randomized runs: %d\n\n", num_runs); - - for (int irun=0; irun(engine); - } - printf("ok!\n"); - - printf("\n"); - -} // TEST_CASE - -} // namespace diff --git a/components/eamxx/src/diagnostics/tests/precip_liq_surf_mass_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/precip_liq_surf_mass_flux_tests.cpp deleted file mode 100644 index bb404cd7f873..000000000000 --- a/components/eamxx/src/diagnostics/tests/precip_liq_surf_mass_flux_tests.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/precip_liq_surf_mass_flux.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/share/physics_constants.hpp" - -#include "share/util/scream_setup_random_test.hpp" -#include "share/field/field_utils.hpp" - -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" - -#include - -namespace scream { - -std::shared_ptr -create_gm (const ekat::Comm& comm, const int ncols) { - - const int num_global_cols = ncols*comm.size(); - - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_vertical_levels", 1); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); - gm->build_grids(); - - return gm; -} - -//-----------------------------------------------------------------------------------------------// -template -void run(std::mt19937_64& engine) -{ - using PC = scream::physics::Constants; - using KT = ekat::KokkosTypes; - - // A world comm - ekat::Comm comm(MPI_COMM_WORLD); - - // Create a grids manager - const int ncols = 13; - auto gm = create_gm(comm,ncols); - - // Create timestep - const int dt=1800; - - // Construct random input data - using RPDF = std::uniform_real_distribution; - RPDF pdf_mass(0.0,1.0); - - // Initial A time stamp - util::TimeStamp t0 ({2022,1,1},{0,0,0}); - - // Construct the Diagnostic - ekat::ParameterList params; - register_diagnostics(); - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - auto diag = diag_factory.create("PrecipLiqSurfMassFlux",comm,params); - diag->set_grids(gm); - - // Set the required fields for the diagnostic. - std::map input_fields; - for (const auto& req : diag->get_required_field_requests()) { - Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(); - f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - f.get_header().get_tracking().set_accum_start_time(t0); - diag->set_required_field(f.get_const()); - REQUIRE_THROWS(diag->set_computed_field(f)); - input_fields.emplace(name,f); - } - - // Initialize the diagnostic - diag->initialize(t0,RunType::Initial); - - // Run tests - { - util::TimeStamp t = t0 + dt; - - // Construct random data to use for test - // Get views of input data and set to random values - auto precip_liq_surf_mass_f = input_fields["precip_liq_surf_mass"]; - const auto& precip_liq_surf_mass_v = precip_liq_surf_mass_f.get_view(); - for (int icol=0;icolcompute_diagnostic(); - const auto& diag_out = diag->get_diagnostic(); - Field theta_f = diag_out.clone(); - theta_f.deep_copy(0.0); - theta_f.sync_to_dev(); - const auto& theta_v = theta_f.get_view(); - const auto rhodt = PC::RHO_H2O*dt; - Kokkos::parallel_for("precip_liq_surf_mass_flux_test", - typename KT::RangePolicy(0,ncols), - KOKKOS_LAMBDA(const int& icol) { - theta_v(icol) = precip_liq_surf_mass_v(icol) / rhodt; - }); - Kokkos::fence(); - REQUIRE(views_are_equal(diag_out,theta_f)); - } - - // Finalize the diagnostic - diag->finalize(); - -} // run() - -TEST_CASE("precip_liq_surf_mass_flux_test", "precip_liq_surf_mass_flux_test]"){ - using scream::Real; - using Device = scream::DefaultDevice; - - constexpr int num_runs = 5; - - auto engine = scream::setup_random_test(); - - printf(" -> Number of randomized runs: %d\n\n", num_runs); - - for (int irun=0; irun(engine); - } - printf("ok!\n"); - - printf("\n"); - -} // TEST_CASE - -} // namespace diff --git a/components/eamxx/src/diagnostics/tests/precip_total_surf_mass_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/precip_surf_mass_flux_tests.cpp similarity index 51% rename from components/eamxx/src/diagnostics/tests/precip_total_surf_mass_flux_tests.cpp rename to components/eamxx/src/diagnostics/tests/precip_surf_mass_flux_tests.cpp index d32143946865..997445860876 100644 --- a/components/eamxx/src/diagnostics/tests/precip_total_surf_mass_flux_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/precip_surf_mass_flux_tests.cpp @@ -1,7 +1,6 @@ #include "catch2/catch.hpp" #include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/precip_total_surf_mass_flux.hpp" #include "diagnostics/register_diagnostics.hpp" #include "physics/share/physics_constants.hpp" @@ -12,8 +11,6 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "ekat/util/ekat_test_utils.hpp" -#include - namespace scream { std::shared_ptr @@ -21,9 +18,14 @@ create_gm (const ekat::Comm& comm, const int ncols) { const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_vertical_levels", 1); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", 1); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); @@ -56,12 +58,15 @@ void run(std::mt19937_64& engine) util::TimeStamp t0 ({2022,1,1},{0,0,0}); // Construct the Diagnostics - ekat::ParameterList params; register_diagnostics(); + ekat::ParameterList params; auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - auto diag_total = diag_factory.create("PrecipTotalSurfMassFlux", comm, params); - auto diag_ice = diag_factory.create("PrecipIceSurfMassFlux", comm, params); - auto diag_liq = diag_factory.create("PrecipLiqSurfMassFlux", comm, params); + params.set("precip_type","total"); + auto diag_total = diag_factory.create("precip_surf_mass_flux", comm, params); + params.set("precip_type","ice"); + auto diag_ice = diag_factory.create("precip_surf_mass_flux" , comm, params); + params.set("precip_type","liq"); + auto diag_liq = diag_factory.create("precip_surf_mass_flux" , comm, params); diag_total->set_grids(gm); diag_ice->set_grids(gm); diag_liq->set_grids(gm); @@ -94,62 +99,49 @@ void run(std::mt19937_64& engine) diag_liq->initialize(t0,RunType::Initial); // Run tests - { - util::TimeStamp t = t0 + dt; - - // Construct random data to use for test - // Get views of input data and set to random values - auto precip_ice_surf_mass_f = input_fields["precip_ice_surf_mass"]; - const auto& precip_ice_surf_mass_v = precip_ice_surf_mass_f.get_view(); - auto precip_liq_surf_mass_f = input_fields["precip_liq_surf_mass"]; - const auto& precip_liq_surf_mass_v = precip_liq_surf_mass_f.get_view(); - for (int icol=0;icol(); + auto precip_liq_surf_mass_f = input_fields["precip_liq_surf_mass"]; + auto precip_liq_surf_mass_v = precip_liq_surf_mass_f.get_view(); + for (int icol=0;icolcompute_diagnostic(); - const auto& diag_total_out = diag_total->get_diagnostic(); - Field theta_f = diag_total_out.clone(); - theta_f.deep_copy(0.0); - theta_f.sync_to_dev(); - const auto& theta_v = theta_f.get_view(); - const auto rhodt = PC::RHO_H2O*dt; - Kokkos::parallel_for("precip_total_surf_mass_flux_test", - typename KT::RangePolicy(0,ncols), - KOKKOS_LAMBDA(const int& icol) { - theta_v(icol) = precip_ice_surf_mass_v(icol)/rhodt + - precip_liq_surf_mass_v(icol)/rhodt; - }); - Kokkos::fence(); - REQUIRE(views_are_equal(diag_total_out,theta_f)); - - // Test against sum of precip_ice/liq diagnostics - diag_ice->compute_diagnostic(); - diag_liq->compute_diagnostic(); - - const auto& diag_ice_out = diag_ice->get_diagnostic(); - const auto& diag_liq_out = diag_liq->get_diagnostic(); - - Field sum_of_diags_f(diag_total_out.get_header().get_identifier()); - sum_of_diags_f.get_header().get_alloc_properties().request_allocation(); - sum_of_diags_f.allocate_view(); - const auto& sum_of_diags_v = sum_of_diags_f.get_view(); - - const auto& diag_ice_v = diag_ice_out.get_view(); - const auto& diag_liq_v = diag_liq_out.get_view(); - Kokkos::parallel_for("calculate_sum_of_diags", - typename KT::RangePolicy(0,ncols), - KOKKOS_LAMBDA(const int& icol) { - sum_of_diags_v(icol) = diag_ice_v(icol) + diag_liq_v(icol); - }); - Kokkos::fence(); - REQUIRE(views_are_equal(diag_total_out,sum_of_diags_f)); -} + precip_ice_surf_mass_f.get_header().get_tracking().update_time_stamp(t); + precip_liq_surf_mass_f.get_header().get_tracking().update_time_stamp(t); + + // Run diagnostics and compare with manual calculation + diag_total->compute_diagnostic(); + diag_liq->compute_diagnostic(); + diag_ice->compute_diagnostic(); + + Field preicp_total_f = diag_total->get_diagnostic().clone(); + Field preicp_liq_f = diag_liq->get_diagnostic().clone(); + Field preicp_ice_f = diag_ice->get_diagnostic().clone(); + preicp_total_f.deep_copy(0.0); + preicp_liq_f.deep_copy(0.0); + preicp_ice_f.deep_copy(0.0); + auto precip_total_v = preicp_total_f.get_view(); + auto precip_liq_v = preicp_liq_f.get_view(); + auto precip_ice_v = preicp_ice_f.get_view(); + const auto rhodt = PC::RHO_H2O*dt; + Kokkos::parallel_for("precip_total_surf_mass_flux_test", + typename KT::RangePolicy(0,ncols), + KOKKOS_LAMBDA(const int& icol) { + precip_liq_v(icol) = precip_liq_surf_mass_v(icol)/rhodt; + precip_ice_v(icol) = precip_ice_surf_mass_v(icol)/rhodt; + precip_total_v(icol) = precip_liq_v(icol) + precip_ice_v(icol); + }); + Kokkos::fence(); + + REQUIRE(views_are_equal(diag_total->get_diagnostic(),preicp_total_f)); + REQUIRE(views_are_equal(diag_liq->get_diagnostic(),preicp_liq_f)); + REQUIRE(views_are_equal(diag_ice->get_diagnostic(),preicp_ice_f)); // Finalize the diagnostic diag_total->finalize(); diff --git a/components/eamxx/src/diagnostics/tests/relative_humidity_tests.cpp b/components/eamxx/src/diagnostics/tests/relative_humidity_tests.cpp index ef2816f1f791..b36c6d1a3f4c 100644 --- a/components/eamxx/src/diagnostics/tests/relative_humidity_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/relative_humidity_tests.cpp @@ -22,16 +22,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); - + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); @@ -65,9 +65,11 @@ void run(std::mt19937_64& engine) // Kokkos Policy auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, num_mid_packs); - // Input (randomized) views + // Input (randomized) views, device view_1d temperature("temperature",num_mid_packs), pressure("pressure",num_mid_packs), + pseudo_density("pseudo_density",num_mid_packs), + pseudo_density_dry("pseudo_density_dry",num_mid_packs), qv("qv",num_mid_packs); auto dview_as_real = [&] (const view_1d& v) -> rview_1d { @@ -76,9 +78,10 @@ void run(std::mt19937_64& engine) // Construct random input data using RPDF = std::uniform_real_distribution; - RPDF pdf_pres(0.0,PC::P0), + RPDF pdf_pres(10.0,PC::P0), pdf_temp(200.0,400.0), - pdf_qv(0.0,1e-2); + pdf_qv(0.0,1e-2), + pdf_pseudo_density(1.0,100.0); // A time stamp util::TimeStamp t0 ({2022,1,1},{0,0,0}); @@ -112,29 +115,46 @@ void run(std::mt19937_64& engine) { // Construct random data to use for test // Get views of input data and set to random values - const auto& T_mid_f = input_fields["T_mid"]; - const auto& T_mid_v = T_mid_f.get_view(); - const auto& p_mid_f = input_fields["p_mid"]; - const auto& p_mid_v = p_mid_f.get_view(); + + // Field + const auto& T_mid_f = input_fields["T_mid"]; + // its device view + const auto& T_mid_v = T_mid_f.get_view(); + + const auto& p_dry_mid_f = input_fields["p_dry_mid"]; + const auto& p_dry_mid_v = p_dry_mid_f.get_view(); + + const auto& dpwet_f = input_fields["pseudo_density"]; + const auto& dpwet_v = dpwet_f.get_view(); + + const auto& dpdry_f = input_fields["pseudo_density_dry"]; + const auto& dpdry_v = dpdry_f.get_view(); + const auto& qv_f = input_fields["qv"]; const auto& qv_v = qv_f.get_view(); for (int icol=0;icolcompute_diagnostic(); - const auto& diag_out = diag->get_diagnostic(); - Field rh_f = diag_out.clone(); + Field rh_f = T_mid_f.clone(); rh_f.deep_copy(0.0); rh_f.sync_to_dev(); const auto& rh_v = rh_f.get_view(); @@ -143,13 +163,30 @@ void run(std::mt19937_64& engine) Smask range_mask(true); Kokkos::parallel_for("", policy, KOKKOS_LAMBDA(const MemberType& team) { const int icol = team.league_rank(); + + const auto& dpwet_sub = ekat::subview(dpwet_v,icol); + const auto& dpdry_sub = ekat::subview(dpdry_v,icol); + const auto& qv_sub = ekat::subview(qv_v,icol); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,num_mid_packs), [&] (const Int& jpack) { - auto qv_sat_l = physics::qv_sat(T_mid_v(icol,jpack), p_mid_v(icol,jpack), false, range_mask); + dpdry_sub(jpack) = dpwet_sub(jpack) - dpwet_sub(jpack)*qv_sub(jpack); + auto qv_sat_l = physics::qv_sat_dry(T_mid_v(icol,jpack), p_dry_mid_v(icol,jpack), false, range_mask); + qv_sat_l *= dpdry_v(icol,jpack) ; + qv_sat_l /= dpwet_v(icol,jpack) ; rh_v(icol,jpack) = qv_v(icol,jpack)/qv_sat_l; }); team.team_barrier(); }); Kokkos::fence(); + + // Run diagnostic and compare with manual calculation + diag->compute_diagnostic(); + const auto& diag_out = diag->get_diagnostic(); + + //in case one needs to look at values + //print_field_hyperslab(rh_f); + //print_field_hyperslab(diag_out); + REQUIRE(views_are_equal(diag_out,rh_f)); } diff --git a/components/eamxx/src/diagnostics/tests/sea_level_pressure_test.cpp b/components/eamxx/src/diagnostics/tests/sea_level_pressure_test.cpp index f64f7c0ea82e..b89d1d20157d 100644 --- a/components/eamxx/src/diagnostics/tests/sea_level_pressure_test.cpp +++ b/components/eamxx/src/diagnostics/tests/sea_level_pressure_test.cpp @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); diff --git a/components/eamxx/src/diagnostics/tests/shortwave_cloud_forcing_tests.cpp b/components/eamxx/src/diagnostics/tests/shortwave_cloud_forcing_tests.cpp index 67ae5e907cba..94d719c1884a 100644 --- a/components/eamxx/src/diagnostics/tests/shortwave_cloud_forcing_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/shortwave_cloud_forcing_tests.cpp @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); diff --git a/components/eamxx/src/diagnostics/tests/surf_upward_latent_heat_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/surf_upward_latent_heat_flux_tests.cpp new file mode 100644 index 000000000000..f233bfaf1713 --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/surf_upward_latent_heat_flux_tests.cpp @@ -0,0 +1,136 @@ +#include "catch2/catch.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" +#include "diagnostics/surf_upward_latent_heat_flux.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "physics/share/physics_constants.hpp" + +#include "share/util/scream_setup_random_test.hpp" +#include "share/field/field_utils.hpp" + +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "ekat/util/ekat_test_utils.hpp" +#include "ekat/logging/ekat_logger.hpp" + +namespace scream { + +std::shared_ptr +create_gm (const ekat::Comm& comm, const int ncols) { + + const int num_global_cols = ncols*comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", 1); + + auto gm = create_mesh_free_grids_manager(comm,gm_params); + gm->build_grids(); + + return gm; +} + +//-----------------------------------------------------------------------------------------------// +template +void run(std::mt19937_64& engine, const ekat::Comm& comm, LoggerType& logger) +{ + using PC = scream::physics::Constants; + using KT = ekat::KokkosTypes; + + // Create a grids manager + const int ncols = 13; + auto gm = create_gm(comm,ncols); + + // Construct random input data + using RPDF = std::uniform_real_distribution; + RPDF pdf_mass(0.0,1.0); + + // Initial time stamp + util::TimeStamp t0 ({2022,1,1},{0,0,0}); + + // Construct the Diagnostics + register_diagnostics(); + ekat::ParameterList params; + + auto& diag_factory = AtmosphereDiagnosticFactory::instance(); + auto diag_latent_heat = diag_factory.create("surface_upward_latent_heat_flux", comm, params); + diag_latent_heat->set_grids(gm); + + // Set the required fields for the diagnostic. + std::map input_fields; + for (const auto& req: diag_latent_heat->get_required_field_requests()) { + Field f(req.fid); + f.get_header().get_alloc_properties().request_allocation(); + f.allocate_view(); + const auto name = f.name(); + f.get_header().get_tracking().update_time_stamp(t0); + diag_latent_heat->set_required_field(f.get_const()); + input_fields.emplace(name, f); + } + + // Initialize the diagnostic + diag_latent_heat->initialize(t0,RunType::Initial); + const auto& surf_evap_f = input_fields["surf_evap"]; + const auto& surf_evap_v = surf_evap_f.get_view(); + for (int icol=0; icolcompute_diagnostic(); + const auto diag_latent_heat_out = diag_latent_heat->get_diagnostic(); + Field surf_lhf = diag_latent_heat_out.clone(); + surf_lhf.deep_copy(0.0); + surf_lhf.sync_to_dev(); + const auto& surf_lhf_v = surf_lhf.get_view(); + constexpr auto latent_heat_evap = PC::LatVap; // [J/kg] + Kokkos::parallel_for("surf_upward_latent_heat_flux_test", + typename KT::RangePolicy(0, ncols), + KOKKOS_LAMBDA (const int& icol) { + surf_lhf_v(icol) = surf_evap_v(icol) * latent_heat_evap; + }); + Kokkos::fence(); + + if (!views_are_equal(diag_latent_heat_out, surf_lhf)) { + // In case of failure, log additional info before aborting with + // Catch2's REQUIRE macro + logger.error("error: surf_lhf_v and diag_latent_heat_out are not passing the views_are_equal test."); + auto surf_lhf_h = Kokkos::create_mirror_view(surf_lhf_v); + diag_latent_heat_out.sync_to_host(); + auto diag_latent_heat_out_h = diag_latent_heat->get_diagnostic().get_view(); + Kokkos::deep_copy(surf_lhf_h, surf_lhf_v); + for (int i=0; ifinalize(); +} // run + +TEST_CASE("surf_upward_latent_heat_flux_test", "[surf_upward_latent_heat_flux_test]") { + using scream::Real; + using Device = scream::DefaultDevice; + + ekat::Comm comm(MPI_COMM_WORLD); + ekat::logger::Logger<> logger("surf_upward_latent_heat_flux_test", + ekat::logger::LogLevel::debug, comm); + constexpr int num_runs = 5; + auto engine = scream::setup_random_test(); + logger.info(" -> Number of randomized runs: {}", num_runs); + for (int irun=0; irun(engine, comm, logger); + } + logger.info("Tests complete."); + logger.info("ok!"); + +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp new file mode 100644 index 000000000000..aceb705a61c7 --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp @@ -0,0 +1,176 @@ +#include "catch2/catch.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" +#include "diagnostics/vapor_flux.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "physics/share/physics_constants.hpp" + +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_common_physics_functions.hpp" +#include "share/field/field_utils.hpp" + +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "ekat/util/ekat_test_utils.hpp" + +#include + +namespace scream { + +std::shared_ptr +create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { + + const int num_global_cols = ncols*comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm,gm_params); + gm->build_grids(); + + return gm; +} + +//-----------------------------------------------------------------------------------------------// +template +void run(std::mt19937_64& engine) +{ + using PC = scream::physics::Constants; + using KT = ekat::KokkosTypes; + using ExecSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + using MemberType = typename KT::MemberType; + using view_1d = typename KT::template view_1d; + + constexpr int num_levs = 33; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // Create a grids manager - single column for these tests + const int ncols = 1; //TODO should be set to the size of the communication group. + auto gm = create_gm(comm,ncols,num_levs); + + // Kokkos Policy + auto policy = ESU::get_default_team_policy(ncols, num_levs); + + // Input (randomized) views + view_1d qv("qv",num_levs), + pseudo_density("pseudo_density",num_levs), + u("u",num_levs), + v("v",num_levs); + + + // Construct random input data + using RPDF = std::uniform_real_distribution; + RPDF pdf_qv(0.0,1e-3), + pdf_uv(0.0,200), + pdf_pseudo_density(1.0,100.0); + + // A time stamp + util::TimeStamp t0 ({2022,1,1},{0,0,0}); + + ekat::ParameterList params; + register_diagnostics(); + auto& diag_factory = AtmosphereDiagnosticFactory::instance(); + + REQUIRE_THROWS (diag_factory.create("VaporFlux",comm,params)); // No 'Wind Component' + params.set("Wind Component","foo"); + REQUIRE_THROWS (diag_factory.create("VaporFlux",comm,params)); // Invalid 'Wind Component' + for (const std::string& which_comp : {"Zonal", "Meridional"}) { + // Construct the Diagnostic + params.set("Wind Component",which_comp); + auto diag = diag_factory.create("VaporFlux",comm,params); + diag->set_grids(gm); + + // Set the required fields for the diagnostic. + std::map input_fields; + for (const auto& req : diag->get_required_field_requests()) { + Field f(req.fid); + f.allocate_view(); + f.get_header().get_tracking().update_time_stamp(t0); + diag->set_required_field(f.get_const()); + input_fields.emplace(f.name(),f); + } + + // Initialize the diagnostic + diag->initialize(t0,RunType::Initial); + + // Run tests + { + // Construct random data to use for test + // Get views of input data and set to random values + const auto& qv_f = input_fields["qv"]; + const auto& pseudo_density_f = input_fields["pseudo_density"]; + const auto& horiz_winds_f = input_fields["horiz_winds"]; + + const auto& qv_v = qv_f.get_view(); + const auto& pseudo_density_v = pseudo_density_f.get_view(); + const auto& horiz_winds_v = horiz_winds_f.get_view(); + + for (int icol=0;icolcompute_diagnostic(); + const auto& diag_out = diag->get_diagnostic(); + Field qv_vert_integrated_flux_u_f = diag_out.clone(); + qv_vert_integrated_flux_u_f.deep_copy(0.0); + qv_vert_integrated_flux_u_f.sync_to_dev(); + const auto& qv_vert_integrated_flux_u_v = qv_vert_integrated_flux_u_f.get_view(); + constexpr Real g = PC::gravit; + int comp = which_comp=="Zonal" ? 0 : 1; + + Kokkos::parallel_for("", policy, KOKKOS_LAMBDA(const MemberType& team) { + const int icol = team.league_rank(); + auto wind = ekat::subview(horiz_winds_v,icol,comp); + Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), + [&] (const int& ilev, Real& lsum) { + lsum += wind(ilev) * qv_v(icol,ilev) * pseudo_density_v(icol,ilev) / g; + },qv_vert_integrated_flux_u_v(icol)); + + }); + Kokkos::fence(); + REQUIRE(views_are_equal(diag_out,qv_vert_integrated_flux_u_f)); + } + + // Finalize the diagnostic + diag->finalize(); + } +} + +TEST_CASE("zonal_vapor_flux_test", "zonal_vapor_flux_test]"){ + using Device = scream::DefaultDevice; + + constexpr int num_runs = 5; + + auto engine = scream::setup_random_test(); + + printf(" -> Number of randomized runs: %d\n\n", num_runs); + + for (int irun=0; irun(engine); + } + printf("ok!\n"); +} + +} // namespace diff --git a/components/eamxx/src/diagnostics/tests/vertical_layer_interface_test.cpp b/components/eamxx/src/diagnostics/tests/vertical_layer_interface_test.cpp deleted file mode 100644 index 339700328a8f..000000000000 --- a/components/eamxx/src/diagnostics/tests/vertical_layer_interface_test.cpp +++ /dev/null @@ -1,186 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/vertical_layer_interface.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/share/physics_constants.hpp" - -#include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "share/field/field_utils.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" - -#include - -namespace scream { - -std::shared_ptr -create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - - const int num_local_elems = 4; - const int np = 4; - const int num_global_cols = ncols*comm.size(); - - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); - gm->build_grids(); - - return gm; -} - -//-----------------------------------------------------------------------------------------------// -template -void run(std::mt19937_64& engine) -{ - using PF = scream::PhysicsFunctions; - using PC = scream::physics::Constants; - using Pack = ekat::Pack; - using KT = ekat::KokkosTypes; - using ExecSpace = typename KT::ExeSpace; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - using rview_1d = typename KT::template view_1d; - - const int packsize = SCREAM_PACK_SIZE; - constexpr int num_levs = packsize*2 + 1; // Number of levels to use for tests, make sure the last pack can also have some empty slots (packsize>1). - const int num_mid_packs = ekat::npack(num_levs); - - // A world comm - ekat::Comm comm(MPI_COMM_WORLD); - - // Create a grids manager - single column for these tests - const int ncols = 1; - auto gm = create_gm(comm,ncols,num_levs); - - // Kokkos Policy - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, num_mid_packs); - - // Input (randomized) views - view_1d temperature("temperature",num_mid_packs), - pseudodensity("pseudodensity",num_mid_packs), - pressure("pressure",num_mid_packs), - watervapor("watervapor",num_mid_packs); - - auto dview_as_real = [&] (const view_1d& v) -> rview_1d { - return rview_1d(reinterpret_cast(v.data()),v.size()*packsize); - }; - - // Construct random input data - using RPDF = std::uniform_real_distribution; - RPDF pdf_qv(1e-6,1e-3), - pdf_pseudodens(1.0,100.0), - pdf_pres(0.0,PC::P0), - pdf_temp(200.0,400.0); - - // A time stamp - util::TimeStamp t0 ({2022,1,1},{0,0,0}); - - // Construct the Diagnostic - ekat::ParameterList params; - register_diagnostics(); - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - auto diag = diag_factory.create("VerticalLayerInterface",comm,params); - diag->set_grids(gm); - - - // Set the required fields for the diagnostic. - std::map input_fields; - for (const auto& req : diag->get_required_field_requests()) { - Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(packsize); - f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - diag->set_required_field(f.get_const()); - REQUIRE_THROWS(diag->set_computed_field(f)); - input_fields.emplace(name,f); - } - - // Initialize the diagnostic - diag->initialize(t0,RunType::Initial); - - // Run tests - { - // Construct random data to use for test - // Get views of input data and set to random values - const auto& T_mid_f = input_fields["T_mid"]; - const auto& T_mid_v = T_mid_f.get_view(); - const auto& pseudo_dens_f = input_fields["pseudo_density"]; - const auto& pseudo_dens_v = pseudo_dens_f.get_view(); - const auto& p_mid_f = input_fields["p_mid"]; - const auto& p_mid_v = p_mid_f.get_view(); - const auto& qv_mid_f = input_fields["qv"]; - const auto& qv_mid_v = qv_mid_f.get_view(); - for (int icol=0;icolcompute_diagnostic(); - const auto& diag_out = diag->get_diagnostic(); - Field zint_f = diag_out.clone(); - zint_f.deep_copy(0.0); - zint_f.sync_to_dev(); - const auto& zint_v = zint_f.get_view(); - view_1d dz_v("",num_mid_packs); - Kokkos::parallel_for("", policy, KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team,num_mid_packs), [&] (const Int& jpack) { - dz_v(jpack) = PF::calculate_dz(pseudo_dens_v(icol,jpack),p_mid_v(icol,jpack),T_mid_v(icol,jpack),qv_mid_v(icol,jpack)); - }); - team.team_barrier(); - const auto& zint_s = ekat::subview(zint_v,icol); - PF::calculate_z_int(team,num_levs,dz_v,0.0,zint_s); - }); - Kokkos::fence(); - REQUIRE(views_are_equal(diag_out,zint_f)); - } - - // Finalize the diagnostic - diag->finalize(); - -} // run() - -TEST_CASE("vertical_layer_interface_test", "vertical_layer_interface_test]"){ - // Run tests for both Real and Pack, and for (potentially) different pack sizes - using scream::Real; - using Device = scream::DefaultDevice; - - constexpr int num_runs = 5; - - auto engine = scream::setup_random_test(); - - printf(" -> Number of randomized runs: %d\n\n", num_runs); - - printf(" -> Testing Pack scalar type...",SCREAM_PACK_SIZE); - for (int irun=0; irun(engine); - } - printf("ok!\n"); - - printf("\n"); - -} // TEST_CASE - -} // namespace diff --git a/components/eamxx/src/diagnostics/tests/vertical_layer_midpoint_test.cpp b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp similarity index 63% rename from components/eamxx/src/diagnostics/tests/vertical_layer_midpoint_test.cpp rename to components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp index 851aa770faa9..32402955f8da 100644 --- a/components/eamxx/src/diagnostics/tests/vertical_layer_midpoint_test.cpp +++ b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp @@ -1,7 +1,7 @@ #include "catch2/catch.hpp" #include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/vertical_layer_midpoint.hpp" +#include "diagnostics/vertical_layer.hpp" #include "diagnostics/register_diagnostics.hpp" #include "physics/share/physics_constants.hpp" @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); @@ -39,7 +40,7 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { //-----------------------------------------------------------------------------------------------// template -void run(std::mt19937_64& engine) +void run(std::mt19937_64& engine, std::string diag_type, const bool from_sea_level = false) { using PF = scream::PhysicsFunctions; using PC = scream::physics::Constants; @@ -49,6 +50,7 @@ void run(std::mt19937_64& engine) using MemberType = typename KT::MemberType; using view_1d = typename KT::template view_1d; using rview_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; const int packsize = SCREAM_PACK_SIZE; constexpr int num_levs = packsize*2 + 1; // Number of levels to use for tests, make sure the last pack can also have some empty slots (packsize>1). @@ -80,18 +82,33 @@ void run(std::mt19937_64& engine) RPDF pdf_qv(1e-6,1e-3), pdf_pseudodens(1.0,100.0), pdf_pres(0.0,PC::P0), - pdf_temp(200.0,400.0); + pdf_temp(200.0,400.0), + pdf_phis(0.0,10000.0); // A time stamp util::TimeStamp t0 ({2022,1,1},{0,0,0}); // Construct the Diagnostic ekat::ParameterList params; + std::string diag_name; + if (diag_type == "thickness") { + diag_name = "dz"; + } + else if (diag_type == "interface") { + diag_name = from_sea_level ? "z_int" : "geopotential_int"; + } else if (diag_type == "midpoint") { + diag_name = from_sea_level ? "z_mid" : "geopotential_mid"; + } + params.set("diag_name", diag_name); register_diagnostics(); auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - auto diag = diag_factory.create("VerticalLayerMidpoint",comm,params); + auto diag = diag_factory.create(diag_name,comm,params); diag->set_grids(gm); + // Helpful bools + const bool only_compute_dz = (diag_type == "thickness"); + const bool is_interface_layout = (diag_type == "interface"); + const bool generate_phis_data = (not only_compute_dz and not from_sea_level); // Set the required fields for the diagnostic. std::map input_fields; @@ -122,6 +139,13 @@ void run(std::mt19937_64& engine) const auto& p_mid_v = p_mid_f.get_view(); const auto& qv_mid_f = input_fields["qv"]; const auto& qv_mid_v = qv_mid_f.get_view(); + Field phis_f; + rview_1d phis_v; + if (generate_phis_data) { + phis_f = input_fields["phis"]; + phis_v = phis_f.get_view(); + } + for (int icol=0;icolcompute_diagnostic(); const auto& diag_out = diag->get_diagnostic(); - Field zmid_f = diag_out.clone(); - zmid_f.deep_copy(0.0); - const auto& zmid_v = zmid_f.get_view(); + // Need to generate temporary values for calculation - const auto& zint_v = view_1d("",num_mid_packs_p1); - const auto& dz_v = view_1d("",num_mid_packs); + const auto& dz_v = view_2d("",ncols, num_mid_packs); + const auto& zmid_v = view_2d("",ncols, num_mid_packs); + const auto& zint_v = view_2d("",ncols, num_mid_packs_p1); + Kokkos::parallel_for("", policy, KOKKOS_LAMBDA(const MemberType& team) { const int icol = team.league_rank(); + + const auto& dz_s = ekat::subview(dz_v,icol); Kokkos::parallel_for(Kokkos::TeamVectorRange(team,num_mid_packs), [&] (const Int& jpack) { - dz_v(jpack) = PF::calculate_dz(pseudo_dens_v(icol,jpack),p_mid_v(icol,jpack),T_mid_v(icol,jpack),qv_mid_v(icol,jpack)); + dz_s(jpack) = PF::calculate_dz(pseudo_dens_v(icol,jpack),p_mid_v(icol,jpack),T_mid_v(icol,jpack),qv_mid_v(icol,jpack)); }); team.team_barrier(); - const auto& zmid_sub = ekat::subview(zmid_v,icol); - PF::calculate_z_int(team,num_levs,dz_v,0.0,zint_v); - PF::calculate_z_mid(team,num_levs,zint_v,zmid_sub); + + if (not only_compute_dz) { + const auto& zint_s = ekat::subview(zint_v,icol); + const Real surf_geopotential = from_sea_level ? 0.0 : phis_v(icol); + PF::calculate_z_int(team,num_levs,dz_s,surf_geopotential,zint_s); + + if (not is_interface_layout) { + const auto& zmid_s = ekat::subview(zmid_v,icol); + PF::calculate_z_mid(team,num_levs,zint_s,zmid_s); + } + } }); Kokkos::fence(); - REQUIRE(views_are_equal(diag_out,zmid_f)); + + Field diag_calc = diag_out.clone(); + auto field_v = diag_calc.get_view(); + + if (diag_type == "thickness") Kokkos::deep_copy(field_v, dz_v); + else if (diag_type == "interface") Kokkos::deep_copy(field_v, zint_v); + else if (diag_type == "midpoint") Kokkos::deep_copy(field_v, zmid_v); + diag_calc.sync_to_host(); + REQUIRE(views_are_equal(diag_out,diag_calc)); } - + // Finalize the diagnostic - diag->finalize(); + diag->finalize(); } // run() -TEST_CASE("vertical_layer_midpoint_test", "vertical_layer_midpoint_test]"){ +TEST_CASE("vertical_layer_test", "vertical_layer_test]"){ // Run tests for both Real and Pack, and for (potentially) different pack sizes using scream::Real; using Device = scream::DefaultDevice; @@ -174,11 +219,21 @@ TEST_CASE("vertical_layer_midpoint_test", "vertical_layer_midpoint_test]"){ auto engine = scream::setup_random_test(); - printf(" -> Number of randomized runs: %d\n\n", num_runs); + printf(" -> Number of randomized runs: %d, Pack scalar type\n\n", num_runs, SCREAM_PACK_SIZE); - printf(" -> Testing Pack scalar type...",SCREAM_PACK_SIZE); + printf(" -> Testing dz..."); + for (int irun=0; irun(engine, "thickness"); + } + printf("ok!\n"); + printf(" -> Testing z_int/geopotential_int..."); + for (int irun=0; irun(engine, "interface", irun%2==0); // alternate from_sea_level=true/false + } + printf("ok!\n"); + printf(" -> Testing z_mid/geopotential_mid..."); for (int irun=0; irun(engine); + run(engine, "midpoint", irun%2==0); // alternate from_sea_level=true/false } printf("ok!\n"); diff --git a/components/eamxx/src/diagnostics/tests/vertical_layer_thickness_test.cpp b/components/eamxx/src/diagnostics/tests/vertical_layer_thickness_test.cpp deleted file mode 100644 index 9762c2955d11..000000000000 --- a/components/eamxx/src/diagnostics/tests/vertical_layer_thickness_test.cpp +++ /dev/null @@ -1,182 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/vertical_layer_thickness.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/share/physics_constants.hpp" - -#include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "share/field/field_utils.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" - -#include - -namespace scream { - -std::shared_ptr -create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - - const int num_local_elems = 4; - const int np = 4; - const int num_global_cols = ncols*comm.size(); - - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); - gm->build_grids(); - - return gm; -} - -//-----------------------------------------------------------------------------------------------// -template -void run(std::mt19937_64& engine) -{ - using PF = scream::PhysicsFunctions; - using PC = scream::physics::Constants; - using Pack = ekat::Pack; - using KT = ekat::KokkosTypes; - using ExecSpace = typename KT::ExeSpace; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - using rview_1d = typename KT::template view_1d; - - const int packsize = SCREAM_PACK_SIZE; - constexpr int num_levs = packsize*2 + 1; // Number of levels to use for tests, make sure the last pack can also have some empty slots (packsize>1). - const int num_mid_packs = ekat::npack(num_levs); - - // A world comm - ekat::Comm comm(MPI_COMM_WORLD); - - // Create a grids manager - single column for these tests - const int ncols = 1; - auto gm = create_gm(comm,ncols,num_levs); - - // Kokkos Policy - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, num_mid_packs); - - // Input (randomized) views - view_1d temperature("temperature",num_mid_packs), - pseudodensity("pseudodensity",num_mid_packs), - pressure("pressure",num_mid_packs), - watervapor("watervapor",num_mid_packs); - - auto dview_as_real = [&] (const view_1d& v) -> rview_1d { - return rview_1d(reinterpret_cast(v.data()),v.size()*packsize); - }; - - // Construct random input data - using RPDF = std::uniform_real_distribution; - RPDF pdf_qv(1e-6,1e-3), - pdf_pseudodens(1.0,100.0), - pdf_pres(0.0,PC::P0), - pdf_temp(200.0,400.0); - - // A time stamp - util::TimeStamp t0 ({2022,1,1},{0,0,0}); - - // Construct the Diagnostic - ekat::ParameterList params; - register_diagnostics(); - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - auto diag = diag_factory.create("VerticalLayerThickness",comm,params); - diag->set_grids(gm); - - - // Set the required fields for the diagnostic. - std::map input_fields; - for (const auto& req : diag->get_required_field_requests()) { - Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(packsize); - f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - diag->set_required_field(f.get_const()); - REQUIRE_THROWS(diag->set_computed_field(f)); - input_fields.emplace(name,f); - } - - // Initialize the diagnostic - diag->initialize(t0,RunType::Initial); - - // Run tests - { - // Construct random data to use for test - // Get views of input data and set to random values - const auto& T_mid_f = input_fields["T_mid"]; - const auto& T_mid_v = T_mid_f.get_view(); - const auto& pseudo_dens_f = input_fields["pseudo_density"]; - const auto& pseudo_dens_v = pseudo_dens_f.get_view(); - const auto& p_mid_f = input_fields["p_mid"]; - const auto& p_mid_v = p_mid_f.get_view(); - const auto& qv_mid_f = input_fields["qv"]; - const auto& qv_mid_v = qv_mid_f.get_view(); - for (int icol=0;icolcompute_diagnostic(); - const auto& diag_out = diag->get_diagnostic(); - Field dz_f = diag_out.clone(); - dz_f.deep_copy(0.0); - const auto& dz_v = dz_f.get_view(); - Kokkos::parallel_for("", policy, KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team,num_mid_packs), [&] (const Int& jpack) { - dz_v(icol,jpack) = PF::calculate_dz(pseudo_dens_v(icol,jpack),p_mid_v(icol,jpack),T_mid_v(icol,jpack),qv_mid_v(icol,jpack)); - }); - team.team_barrier(); - }); - Kokkos::fence(); - REQUIRE(views_are_equal(diag_out,dz_f)); - } - - // Finalize the diagnostic - diag->finalize(); - -} // run() - -TEST_CASE("vertical_layer_thickness_test", "vertical_layer_thickness_test]"){ - // Run tests for both Real and Pack, and for (potentially) different pack sizes - using scream::Real; - using Device = scream::DefaultDevice; - - constexpr int num_runs = 5; - - auto engine = scream::setup_random_test(); - - printf(" -> Number of randomized runs: %d\n\n", num_runs); - - printf(" -> Testing Pack scalar type...",SCREAM_PACK_SIZE); - for (int irun=0; irun(engine); - } - printf("ok!\n"); - - printf("\n"); - -} // TEST_CASE - -} // namespace diff --git a/components/eamxx/src/diagnostics/tests/virtual_temperature_test.cpp b/components/eamxx/src/diagnostics/tests/virtual_temperature_test.cpp index c479d10c4dc7..74a934a8c327 100644 --- a/components/eamxx/src/diagnostics/tests/virtual_temperature_test.cpp +++ b/components/eamxx/src/diagnostics/tests/virtual_temperature_test.cpp @@ -21,15 +21,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); diff --git a/components/eamxx/src/diagnostics/tests/water_path_tests.cpp b/components/eamxx/src/diagnostics/tests/water_path_tests.cpp index cb3acffb35b2..f78615f0cfae 100644 --- a/components/eamxx/src/diagnostics/tests/water_path_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/water_path_tests.cpp @@ -1,16 +1,15 @@ #include "catch2/catch.hpp" #include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/rain_water_path.hpp" #include "diagnostics/register_diagnostics.hpp" #include "physics/share/physics_constants.hpp" +#include "share/util/scream_utils.hpp" #include "share/util/scream_setup_random_test.hpp" #include "share/util/scream_common_physics_functions.hpp" #include "share/field/field_utils.hpp" -#include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "ekat/util/ekat_test_utils.hpp" @@ -21,15 +20,16 @@ namespace scream { std::shared_ptr create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - const int num_local_elems = 4; - const int np = 4; const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); @@ -37,29 +37,17 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { return gm; } -template -typename VT::HostMirror -cmvdc (const VT& v) { - auto vh = Kokkos::create_mirror_view(v); - Kokkos::deep_copy(vh,v); - return vh; -} - //-----------------------------------------------------------------------------------------------// template void run(std::mt19937_64& engine) { using PC = scream::physics::Constants; - using Pack = ekat::Pack; using KT = ekat::KokkosTypes; - using ExecSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - using rview_1d = typename KT::template view_1d; + using view_1d = typename KT::template view_1d; - const int packsize = SCREAM_PACK_SIZE; - constexpr int num_levs = packsize*2 + 1; // Number of levels to use for tests, make sure the last pack can also have some empty slots (packsize>1). - const int num_mid_packs = ekat::npack(num_levs); + constexpr int num_levs = 33; constexpr Real gravit = PC::gravit; constexpr Real macheps = PC::macheps; @@ -71,20 +59,16 @@ void run(std::mt19937_64& engine) auto gm = create_gm(comm,ncols,num_levs); // Kokkos Policy - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, num_mid_packs); + auto policy = ESU::get_default_team_policy(ncols, num_levs); // Input (randomized) views view_1d - pseudo_density("pseudo_density",num_mid_packs), - qv("qv",num_mid_packs), - qc("qc",num_mid_packs), - qr("qr",num_mid_packs), - qi("qi",num_mid_packs), - qm("qm",num_mid_packs); - - auto dview_as_real = [&] (const view_1d& v) -> rview_1d { - return rview_1d(reinterpret_cast(v.data()),v.size()*packsize); - }; + pseudo_density("pseudo_density",num_levs), + qv("qv",num_levs), + qc("qc",num_levs), + qr("qr",num_levs), + qi("qi",num_levs), + qm("qm",num_levs); // Construct random input data using RPDF = std::uniform_real_distribution; @@ -102,28 +86,37 @@ void run(std::mt19937_64& engine) auto& diag_factory = AtmosphereDiagnosticFactory::instance(); register_diagnostics(); ekat::ParameterList params; + + REQUIRE_THROWS (diag_factory.create("WaterPath",comm,params)); // No 'Water Kind' + params.set("Water Kind","Foo"); + REQUIRE_THROWS (diag_factory.create("WaterPath",comm,params)); // Invalid 'Water Kind' + // Vapor - auto diag_vap = diag_factory.create("VapWaterPath",comm,params); + params.set("Water Kind","Vap"); + auto diag_vap = diag_factory.create("WaterPath",comm,params); diag_vap->set_grids(gm); diags.emplace("vwp",diag_vap); // Liquid - auto diag_liq = diag_factory.create("LiqWaterPath",comm,params); + params.set("Water Kind","Liq"); + auto diag_liq = diag_factory.create("WaterPath",comm,params); diag_liq->set_grids(gm); diags.emplace("lwp",diag_liq); // Ice - auto diag_ice = diag_factory.create("IceWaterPath",comm,params); + params.set("Water Kind","Ice"); + auto diag_ice = diag_factory.create("WaterPath",comm,params); diag_ice->set_grids(gm); diags.emplace("iwp",diag_ice); // Rime - auto diag_rime = diag_factory.create("RimeWaterPath",comm,params); + params.set("Water Kind","Rime"); + auto diag_rime = diag_factory.create("WaterPath",comm,params); diag_rime->set_grids(gm); diags.emplace("mwp",diag_rime); // Rain - auto diag_rain = diag_factory.create("RainWaterPath",comm,params); + params.set("Water Kind","Rain"); + auto diag_rain = diag_factory.create("WaterPath",comm,params); diag_rain->set_grids(gm); diags.emplace("rwp",diag_rain); - // Set the required fields for the diagnostic. std::map input_fields; for (const auto& dd : diags) { @@ -131,22 +124,13 @@ void run(std::mt19937_64& engine) for (const auto& req : diag->get_required_field_requests()) { if (input_fields.find(req.fid.name())==input_fields.end()) { Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(packsize); f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - diag->set_required_field(f.get_const()); - REQUIRE_THROWS(diag->set_computed_field(f)); - input_fields.emplace(name,f); - } else { - auto& f = input_fields[req.fid.name()]; - const auto name = f.name(); f.get_header().get_tracking().update_time_stamp(t0); - diag->set_required_field(f.get_const()); - REQUIRE_THROWS(diag->set_computed_field(f)); - input_fields.emplace(name,f); + input_fields.emplace(f.name(),f); } + const auto& f = input_fields.at(req.fid.name()); + diag->set_required_field(f.get_const()); + REQUIRE_THROWS(diag->set_computed_field(f)); } // Initialize the diagnostic diag->initialize(t0,RunType::Initial); @@ -157,18 +141,18 @@ void run(std::mt19937_64& engine) { // Construct random data to use for test // Get views of input data and set to random values - const auto& qv_f = input_fields["qv"]; - const auto& qv_v = qv_f.get_view(); - const auto& qc_f = input_fields["qc"]; - const auto& qc_v = qc_f.get_view(); - const auto& qi_f = input_fields["qi"]; - const auto& qi_v = qi_f.get_view(); - const auto& qm_f = input_fields["qm"]; - const auto& qm_v = qm_f.get_view(); - const auto& qr_f = input_fields["qr"]; - const auto& qr_v = qr_f.get_view(); - const auto& pseudo_dens_f = input_fields["pseudo_density"]; - const auto& pseudo_dens_v = pseudo_dens_f.get_view(); + const auto& qv_f = input_fields.at("qv"); + const auto& qv_v = qv_f.get_view(); + const auto& qc_f = input_fields.at("qc"); + const auto& qc_v = qc_f.get_view(); + const auto& qi_f = input_fields.at("qi"); + const auto& qi_v = qi_f.get_view(); + const auto& qm_f = input_fields.at("qm"); + const auto& qm_v = qm_f.get_view(); + const auto& qr_f = input_fields.at("qr"); + const auto& qr_v = qr_f.get_view(); + const auto& pseudo_dens_f = input_fields.at("pseudo_density"); + const auto& pseudo_dens_v = pseudo_dens_f.get_view(); for (int icol=0;icol lmax) { - lmax = qv_s(idx)*dp_s(idx)/gravit; + if (qv_icol(idx)*dp_icol(idx)/gravit > lmax) { + lmax = qv_icol(idx)*dp_icol(idx)/gravit; } }, Kokkos::Max(qv_mass_max)); EKAT_KERNEL_REQUIRE(vwp_v(icol)>qv_mass_max); + team.team_barrier(); // Test qc mass Real qc_mass_max=0.0; - auto qc_s = ekat::scalarize(ekat::subview(qc_v, icol)); + auto qc_icol = ekat::subview(qc_v, icol); Kokkos::parallel_reduce( Kokkos::TeamVectorRange(team,num_levs), [&] (Int idx, Real& lmax) { - if (qc_s(idx)*dp_s(idx)/gravit > lmax) { - lmax = qc_s(idx)*dp_s(idx)/gravit; + if (qc_icol(idx)*dp_icol(idx)/gravit > lmax) { + lmax = qc_icol(idx)*dp_icol(idx)/gravit; } }, Kokkos::Max(qc_mass_max)); EKAT_KERNEL_REQUIRE(lwp_v(icol)>qc_mass_max); + team.team_barrier(); // Test qi mass Real qi_mass_max=0.0; - auto qi_s = ekat::scalarize(ekat::subview(qi_v, icol)); + auto qi_icol = ekat::subview(qi_v, icol); Kokkos::parallel_reduce( Kokkos::TeamVectorRange(team,num_levs), [&] (Int idx, Real& lmax) { - if (qi_s(idx)*dp_s(idx)/gravit > lmax) { - lmax = qi_s(idx)*dp_s(idx)/gravit; + if (qi_icol(idx)*dp_icol(idx)/gravit > lmax) { + lmax = qi_icol(idx)*dp_icol(idx)/gravit; } }, Kokkos::Max(qi_mass_max)); EKAT_KERNEL_REQUIRE(iwp_v(icol)>qi_mass_max); + team.team_barrier(); // Test qm mass Real qm_mass_max=0.0; - auto qm_s = ekat::scalarize(ekat::subview(qm_v, icol)); + auto qm_icol = ekat::subview(qm_v, icol); Kokkos::parallel_reduce( Kokkos::TeamVectorRange(team,num_levs), [&] (Int idx, Real& lmax) { - if (qm_s(idx)*dp_s(idx)/gravit > lmax) { - lmax = qm_s(idx)*dp_s(idx)/gravit; + if (qm_icol(idx)*dp_icol(idx)/gravit > lmax) { + lmax = qm_icol(idx)*dp_icol(idx)/gravit; } }, Kokkos::Max(qm_mass_max)); EKAT_KERNEL_REQUIRE(mwp_v(icol)>qm_mass_max); - + team.team_barrier(); // Test qr mass Real qr_mass_max=0.0; - auto qr_s = ekat::scalarize(ekat::subview(qr_v, icol)); + auto qr_icol = ekat::subview(qr_v, icol); Kokkos::parallel_reduce( Kokkos::TeamVectorRange(team,num_levs), [&] (Int idx, Real& lmax) { - if (qr_s(idx)*dp_s(idx)/gravit > lmax) { - lmax = qr_s(idx)*dp_s(idx)/gravit; + if (qr_icol(idx)*dp_icol(idx)/gravit > lmax) { + lmax = qr_icol(idx)*dp_icol(idx)/gravit; } }, Kokkos::Max(qr_mass_max)); EKAT_KERNEL_REQUIRE(rwp_v(icol)>qr_mass_max); @@ -291,17 +278,18 @@ void run(std::mt19937_64& engine) const auto alpha_qr = pdf_alpha(engine); REQUIRE(alpha_qv*alpha_qc*alpha_qi*alpha_qr != 1.0); - Kokkos::parallel_for("",ncols*num_mid_packs,KOKKOS_LAMBDA(const int& idx) { - const int icol = idx / num_mid_packs; - const int jpack = idx % num_mid_packs; + Kokkos::parallel_for("",ncols*num_levs, + KOKKOS_LAMBDA(const int& idx) { + const int icol = idx / num_levs; + const int ilev = idx % num_levs; - qv_v(icol,jpack) *= alpha_qv; - qc_v(icol,jpack) *= alpha_qc; - qi_v(icol,jpack) *= alpha_qi; - qm_v(icol,jpack) *= alpha_qm; - qr_v(icol,jpack) *= alpha_qr; + qv_v(icol,ilev) *= alpha_qv; + qc_v(icol,ilev) *= alpha_qc; + qi_v(icol,ilev) *= alpha_qi; + qm_v(icol,ilev) *= alpha_qm; + qr_v(icol,ilev) *= alpha_qr; - if (jpack==0) { + if (ilev==0) { vwp_copy_v(icol) *= alpha_qv; lwp_copy_v(icol) *= alpha_qc; iwp_copy_v(icol) *= alpha_qi; @@ -339,31 +327,32 @@ void run(std::mt19937_64& engine) // Test 3: If mass moves from one phase to another than the total water path // should remain unchanged. { - rview_1d total_mass("",ncols); + view_1d total_mass("",ncols); const auto alpha_qv_to_qc = pdf_alpha(engine); const auto alpha_qc_to_qi = pdf_alpha(engine); const auto alpha_qi_to_qr = pdf_alpha(engine); const auto alpha_qr_to_qv = pdf_alpha(engine); - Kokkos::parallel_for("",ncols*num_mid_packs,KOKKOS_LAMBDA(const int& idx) { - const int icol = idx / num_mid_packs; - const int jpack = idx % num_mid_packs; + Kokkos::parallel_for("",ncols*num_levs, + KOKKOS_LAMBDA(const int& idx) { + const int icol = idx / num_levs; + const int ilev = idx % num_levs; total_mass(icol) = vwp_v(icol) + lwp_v(icol) + iwp_v(icol) + rwp_v(icol); - auto qv_to_qc = alpha_qv_to_qc*qv_v(icol,jpack); - auto qc_to_qi = alpha_qc_to_qi*qc_v(icol,jpack); - auto qi_to_qr = alpha_qi_to_qr*qi_v(icol,jpack); - auto qr_to_qv = alpha_qr_to_qv*qr_v(icol,jpack); + auto qv_to_qc = alpha_qv_to_qc*qv_v(icol,ilev); + auto qc_to_qi = alpha_qc_to_qi*qc_v(icol,ilev); + auto qi_to_qr = alpha_qi_to_qr*qi_v(icol,ilev); + auto qr_to_qv = alpha_qr_to_qv*qr_v(icol,ilev); - qv_v(icol,jpack) -= qv_to_qc; - qc_v(icol,jpack) -= qc_to_qi; - qi_v(icol,jpack) -= qi_to_qr; - qr_v(icol,jpack) -= qr_to_qv; + qv_v(icol,ilev) -= qv_to_qc; + qc_v(icol,ilev) -= qc_to_qi; + qi_v(icol,ilev) -= qi_to_qr; + qr_v(icol,ilev) -= qr_to_qv; - qv_v(icol,jpack) += qr_to_qv; - qc_v(icol,jpack) += qv_to_qc; - qi_v(icol,jpack) += qc_to_qi; - qr_v(icol,jpack) += qi_to_qr; + qv_v(icol,ilev) += qr_to_qv; + qc_v(icol,ilev) += qv_to_qc; + qi_v(icol,ilev) += qc_to_qi; + qr_v(icol,ilev) += qi_to_qr; }); Kokkos::fence(); for (const auto& dd : diags) { @@ -384,10 +373,9 @@ void run(std::mt19937_64& engine) const auto alpha_qc_precip = pdf_alpha(engine); const auto alpha_qi_precip = pdf_alpha(engine); const auto alpha_qr_precip = pdf_alpha(engine); - const int surf_pack = num_mid_packs-1; - const int surf_lev = std::max(0,num_levs % Pack::n - 1); - rview_1d total_mass("",ncols); - rview_1d delta_mass("",ncols); + const int surf_lev = num_levs-1; + view_1d total_mass("",ncols); + view_1d delta_mass("",ncols); Kokkos::parallel_for("",ncols,KOKKOS_LAMBDA(const int& icol) { total_mass(icol) = vwp_v(icol) + lwp_v(icol) + iwp_v(icol) + rwp_v(icol); @@ -397,15 +385,15 @@ void run(std::mt19937_64& engine) const auto& qr_sub = ekat::subview(qr_v,icol); const auto& dp_sub = ekat::subview(pseudo_dens_v,icol); - const Real dp_surf = dp_sub(surf_pack)[surf_lev]; + const Real dp_surf = dp_sub(surf_lev); - const Real qc_precip = alpha_qc_precip * qc_sub(surf_pack)[surf_lev]; - const Real qi_precip = alpha_qi_precip * qi_sub(surf_pack)[surf_lev]; - const Real qr_precip = alpha_qr_precip * qr_sub(surf_pack)[surf_lev]; + const Real qc_precip = alpha_qc_precip * qc_sub(surf_lev); + const Real qi_precip = alpha_qi_precip * qi_sub(surf_lev); + const Real qr_precip = alpha_qr_precip * qr_sub(surf_lev); - qc_sub(surf_pack)[surf_lev] -= qc_precip; - qi_sub(surf_pack)[surf_lev] -= qi_precip; - qr_sub(surf_pack)[surf_lev] -= qr_precip; + qc_sub(surf_lev) -= qc_precip; + qi_sub(surf_lev) -= qi_precip; + qr_sub(surf_lev) -= qr_precip; delta_mass(icol) = -(qc_precip + qi_precip + qr_precip) * dp_surf/gravit; @@ -430,17 +418,16 @@ void run(std::mt19937_64& engine) // X*sum(k=0,k=N-1)[k+1] = X*(N-1)*N/2 { Kokkos::deep_copy(pseudo_dens_v,gravit); - Kokkos::parallel_for("",ncols*num_levs,KOKKOS_LAMBDA(const int& idx) { + Kokkos::parallel_for("",ncols*num_levs, + KOKKOS_LAMBDA(const int& idx) { const int icol = idx / num_levs; const int ilev = idx % num_levs; - const int kpack = ilev / Pack::n; - const int klev = ilev % Pack::n; - - qv_v(icol,kpack)[klev] = (icol+1) * (idx+1); - qc_v(icol,kpack)[klev] = (icol+1) * (idx+1); - qi_v(icol,kpack)[klev] = (icol+1) * (idx+1); - qm_v(icol,kpack)[klev] = (icol+1) * (idx+1); - qr_v(icol,kpack)[klev] = (icol+1) * (idx+1); + + qv_v(icol,ilev) = (icol+1) * (idx+1); + qc_v(icol,ilev) = (icol+1) * (idx+1); + qi_v(icol,ilev) = (icol+1) * (idx+1); + qm_v(icol,ilev) = (icol+1) * (idx+1); + qr_v(icol,ilev) = (icol+1) * (idx+1); }); Kokkos::fence(); for (const auto& dd : diags) { @@ -457,7 +444,6 @@ void run(std::mt19937_64& engine) REQUIRE(rwp_h(icol) == (icol+1)*num_levs*(num_levs+1)/2); } } - } // Finalize the diagnostic @@ -469,8 +455,6 @@ void run(std::mt19937_64& engine) } // run() TEST_CASE("water_path_test", "water_path_test]"){ - // Run tests for both Real and Pack, and for (potentially) different pack sizes - using scream::Real; using Device = scream::DefaultDevice; constexpr int num_runs = 5; @@ -479,14 +463,9 @@ TEST_CASE("water_path_test", "water_path_test]"){ printf(" -> Number of randomized runs: %d\n\n", num_runs); - printf(" -> Testing Pack scalar type...",SCREAM_PACK_SIZE); for (int irun=0; irun(engine); } - printf("ok!\n"); - - printf("\n"); - -} // TEST_CASE +} } // namespace diff --git a/components/eamxx/src/diagnostics/tests/zonal_vapor_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/zonal_vapor_flux_tests.cpp deleted file mode 100644 index 5b5182fad982..000000000000 --- a/components/eamxx/src/diagnostics/tests/zonal_vapor_flux_tests.cpp +++ /dev/null @@ -1,184 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/zonal_vapor_flux.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/share/physics_constants.hpp" - -#include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "share/field/field_utils.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" - -#include - -namespace scream { - -std::shared_ptr -create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { - - const int num_local_elems = 4; - const int np = 4; - const int num_global_cols = ncols*comm.size(); - - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); - gm->build_grids(); - - return gm; -} - -//-----------------------------------------------------------------------------------------------// -template -void run(std::mt19937_64& engine) -{ - using Pack = ekat::Pack; - using KT = ekat::KokkosTypes; - using ExecSpace = typename KT::ExeSpace; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - using rview_1d = typename KT::template view_1d; - - const int packsize = SCREAM_PACK_SIZE; - constexpr int num_levs = packsize*2 + 1; // Number of levels to use for tests, make sure the last pack can also have some empty slots (packsize>1). - const int num_mid_packs = ekat::npack(num_levs); - - // A world comm - ekat::Comm comm(MPI_COMM_WORLD); - - // Create a grids manager - single column for these tests - const int ncols = 1; //TODO should be set to the size of the communication group. - auto gm = create_gm(comm,ncols,num_levs); - - // Kokkos Policy - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, num_mid_packs); - - // Input (randomized) views - view_1d qv("qv",num_mid_packs), - pseudo_density("pseudo_density",num_mid_packs), - u("u",num_mid_packs); - - - auto dview_as_real = [&] (const view_1d& v) -> rview_1d { - return rview_1d(reinterpret_cast(v.data()),v.size()*packsize); - }; - - // Construct random input data - using RPDF = std::uniform_real_distribution; - RPDF pdf_qv(0.0,1e-3), - pdf_u(0.0,200), - pdf_pseudo_density(1.0,100.0); - - // A time stamp - util::TimeStamp t0 ({2022,1,1},{0,0,0}); - - // Construct the Diagnostic - ekat::ParameterList params; - register_diagnostics(); - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - auto diag = diag_factory.create("ZonalVapFlux",comm,params); - diag->set_grids(gm); - - - // Set the required fields for the diagnostic. - std::map input_fields; - for (const auto& req : diag->get_required_field_requests()) { - Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(packsize); - f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - diag->set_required_field(f.get_const()); - input_fields.emplace(name,f); - } - - // Initialize the diagnostic - diag->initialize(t0,RunType::Initial); - - // Run tests - { - // Construct random data to use for test - // Get views of input data and set to random values - const auto& qv_f = input_fields["qv"]; - const auto& qv_v = qv_f.get_view(); - const auto& pseudo_density_f = input_fields["pseudo_density"]; - const auto& pseudo_density_v = pseudo_density_f.get_view(); - const auto& horiz_winds_f = input_fields["horiz_winds"]; - const auto& horiz_winds_v = horiz_winds_f.get_view(); - - - for (int icol=0;icol::quiet_NaN()); - - } - - // Run diagnostic and compare with manual calculation - diag->compute_diagnostic(); - const auto& diag_out = diag->get_diagnostic(); - Field qv_vert_integrated_flux_u_f = diag_out.clone(); - qv_vert_integrated_flux_u_f.deep_copy(0.0); - qv_vert_integrated_flux_u_f.sync_to_dev(); - using PC = scream::physics::Constants; - const auto& qv_vert_integrated_flux_u_v = qv_vert_integrated_flux_u_f.get_view(); - constexpr Real gravit = PC::gravit; - Kokkos::parallel_for("", policy, KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - lsum += horiz_winds_v(icol,0,jpack)[klev] * qv_v(icol,jpack)[klev] * pseudo_density_v(icol,jpack)[klev]/gravit; - },qv_vert_integrated_flux_u_v(icol)); - - }); - Kokkos::fence(); - REQUIRE(views_are_equal(diag_out,qv_vert_integrated_flux_u_f)); - } - - // Finalize the diagnostic - diag->finalize(); - -} // run() - -TEST_CASE("zonal_vapor_flux_test", "zonal_vapor_flux_test]"){ - // Run tests for both Real and Pack, and for (potentially) different pack sizes - using scream::Real; - using Device = scream::DefaultDevice; - - constexpr int num_runs = 5; - - auto engine = scream::setup_random_test(); - - printf(" -> Number of randomized runs: %d\n\n", num_runs); - - printf(" -> Testing Pack scalar type...",SCREAM_PACK_SIZE); - for (int irun=0; irun(engine); - } - printf("ok!\n"); - - printf("\n"); - -} // TEST_CASE - -} // namespace diff --git a/components/eamxx/src/diagnostics/vapor_flux.cpp b/components/eamxx/src/diagnostics/vapor_flux.cpp new file mode 100644 index 000000000000..576dd41a1fb5 --- /dev/null +++ b/components/eamxx/src/diagnostics/vapor_flux.cpp @@ -0,0 +1,97 @@ +#include "diagnostics/vapor_flux.hpp" +#include "physics/share/physics_constants.hpp" + +#include + +namespace scream +{ + +VaporFluxDiagnostic:: +VaporFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) +{ + EKAT_REQUIRE_MSG (params.isParameter("Wind Component"), + "Error! VaporFluxDiagnostic requires 'Wind Component' in its input parameters.\n"); + + const auto& comp = m_params.get("Wind Component"); + if (comp=="Zonal") { + m_component = 0; + } else if (comp=="Meridional") { + m_component = 1; + } else { + EKAT_ERROR_MSG ( + "Error! Invalid choice for 'Wind Component' in VaporFluxDiagnostic.\n" + " - input value: " + comp + "\n" + " - valid values: Zonal, Meridional\n"); + } +} + +std::string VaporFluxDiagnostic::name() const +{ + return m_component==0 ? "ZonalVapFlux" : "MeridionalVapFlux"; +} + +void VaporFluxDiagnostic::set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + auto Q = kg/kg; + Q.set_string("kg/kg"); + + auto vel = m/s; + vel.set_string("m/s"); + + auto grid = grids_manager->get_grid("Physics"); + const auto& grid_name = grid->name(); + m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column + + auto scalar2d = grid->get_2d_scalar_layout(); + auto scalar3d = grid->get_3d_scalar_layout(true); + auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + + // The fields required for this diagnostic to be computed + add_field("pseudo_density", scalar3d, Pa, grid_name); + add_field("qv", scalar3d, Q, grid_name); + add_field("horiz_winds", vector3d, m/s, grid_name); + + // Construct and allocate the diagnostic field + FieldIdentifier fid (name(), scalar2d, kg/m/s, grid_name); + m_diagnostic_output = Field(fid); + m_diagnostic_output.allocate_view(); +} + +void VaporFluxDiagnostic::compute_diagnostic_impl() +{ + using PC = scream::physics::Constants; + using KT = KokkosTypes; + using MT = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + + constexpr Real g = PC::gravit; + + const auto diag = m_diagnostic_output.get_view(); + const auto qv = get_field_in("qv").get_view(); + const auto rho = get_field_in("pseudo_density").get_view(); + const auto wind = get_field_in("horiz_winds").get_component(m_component).get_view(); + + const auto num_levs = m_num_levs; + const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); + Kokkos::parallel_for("Compute " + name(), policy, + KOKKOS_LAMBDA(const MT& team) { + const int icol = team.league_rank(); + + auto qv_icol = ekat::subview(qv,icol); + auto rho_icol = ekat::subview(rho,icol); + auto wind_icol = ekat::subview(wind,icol); + + Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), + [&] (const int& ilev, Real& lsum) { + lsum += wind_icol(ilev) * qv_icol(ilev) * rho_icol(ilev) / g; + },diag(icol)); + team.team_barrier(); + }); +} + +} //namespace scream diff --git a/components/eamxx/src/diagnostics/vapor_flux.hpp b/components/eamxx/src/diagnostics/vapor_flux.hpp new file mode 100644 index 000000000000..5c19c9c87499 --- /dev/null +++ b/components/eamxx/src/diagnostics/vapor_flux.hpp @@ -0,0 +1,44 @@ +#ifndef EAMXX_VAPOR_FLUX_DIAGNOSTIC_HPP +#define EAMXX_VAPOR_FLUX_DIAGNOSTIC_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream +{ + +/* + * This diagnostic will produce the zonal or meridional water vapor flux. + */ + +class VaporFluxDiagnostic : public AtmosphereDiagnostic +{ +public: + // Constructors + VaporFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); + + // Set type to diagnostic + AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } + + // The name of the diagnostic + std::string name () const; + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + +protected: +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void compute_diagnostic_impl (); +protected: + + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + int m_component; +}; + +} //namespace scream + +#endif // EAMXX_VAPOR_FLUX_HPP diff --git a/components/eamxx/src/diagnostics/vapor_water_path.cpp b/components/eamxx/src/diagnostics/vapor_water_path.cpp deleted file mode 100644 index f9b07678f8d4..000000000000 --- a/components/eamxx/src/diagnostics/vapor_water_path.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "diagnostics/vapor_water_path.hpp" - -namespace scream -{ - -// ========================================================================================= -VapWaterPathDiagnostic::VapWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void VapWaterPathDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - - // Construct and allocate the diagnostic field - const auto m2 = m*m; - FieldIdentifier fid (name(), scalar2d_layout_mid, kg/m2, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void VapWaterPathDiagnostic::compute_diagnostic_impl() -{ - - using PC = scream::physics::Constants; - constexpr Real gravit = PC::gravit; - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, npacks); - const auto& lwp = m_diagnostic_output.get_view(); - const auto& qv_mid = get_field_in("qv").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - - const auto num_levs = m_num_levs; - Kokkos::parallel_for("VapWaterPathDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - lsum += qv_mid(icol,jpack)[klev] * pseudo_density_mid(icol,jpack)[klev]/gravit; - },lwp(icol)); - team.team_barrier(); - }); - - const auto ts = get_field_in("qv").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/vapor_water_path.hpp b/components/eamxx/src/diagnostics/vapor_water_path.hpp deleted file mode 100644 index 2478d488bc5a..000000000000 --- a/components/eamxx/src/diagnostics/vapor_water_path.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef EAMXX_VAPOR_WATER_PATH_DIAGNOSTIC_HPP -#define EAMXX_VAPOR_WATER_PATH_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the potential temperature. - */ - -class VapWaterPathDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - - // Constructors - VapWaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "VaporWaterPath"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - -}; // class VapWaterPathDiagnostic - -} //namespace scream - -#endif // EAMXX_VAPOR_WATER_PATH_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/vertical_layer.cpp b/components/eamxx/src/diagnostics/vertical_layer.cpp new file mode 100644 index 000000000000..ad6b785d2af7 --- /dev/null +++ b/components/eamxx/src/diagnostics/vertical_layer.cpp @@ -0,0 +1,139 @@ +#include "diagnostics/vertical_layer.hpp" + +namespace scream +{ + +// ========================================================================================= +VerticalLayerDiagnostic:: +VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) +{ + m_diag_name = params.get("diag_name"); + EKAT_REQUIRE_MSG(m_diag_name == "z_int" or m_diag_name == "z_mid" or + m_diag_name == "geopotential_int" or m_diag_name == "geopotential_mid" or + m_diag_name == "dz", + "Error! VerticalLayerDiagnostic has been given an unknown name: "+m_diag_name+".\n"); + + m_only_compute_dz = (m_diag_name == "dz"); + m_is_interface_layout = m_diag_name.find("_int") != std::string::npos; + + // Whether or not diagnostic is computed from sea level depends on the name. + // "z_" -> from sea level, "geopotential_" -> from topography data. + // This boolean is irrelevant for vertical layer thickness (dz). + m_from_sea_level = m_diag_name.find("z_") != std::string::npos; +} +// ======================================================================================== +void VerticalLayerDiagnostic:: +set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + auto Q = kg/kg; + Q.set_string("kg/kg"); + auto m2 = m*m; + auto s2 = s*s; + + auto grid = grids_manager->get_grid("Physics"); + const auto& grid_name = grid->name(); + m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column + + FieldLayout scalar2d_layout { {COL }, {m_num_cols } }; + FieldLayout scalar3d_layout_mid { {COL,LEV }, {m_num_cols,m_num_levs } }; + FieldLayout scalar3d_layout_int { {COL,ILEV}, {m_num_cols,m_num_levs+1} }; + constexpr int ps = Pack::n; + + // The fields required for this diagnostic to be computed + add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); + add_field("qv", scalar3d_layout_mid, Q, grid_name, ps); + + // Only need phis if computing geopotential_* + if (not m_only_compute_dz and not m_from_sea_level) { + add_field("phis", scalar2d_layout, m2/s2, grid_name); + } + + // Construct and allocate the diagnostic field based on the diagnostic name. + const auto diag_layout = m_is_interface_layout ? scalar3d_layout_int : scalar3d_layout_mid; + FieldIdentifier fid (name(), diag_layout, m, grid_name); + m_diagnostic_output = Field(fid); + auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); + C_ap.request_allocation(ps); + m_diagnostic_output.allocate_view(); + + // Initialize temporary views based on need. + if (not m_only_compute_dz) { + if (m_is_interface_layout) { + const auto npacks = ekat::npack(m_num_levs); + m_tmp_midpoint_view = view_2d("tmp_mid",m_num_cols,npacks); + } else { + const auto npacks_p1 = ekat::npack(m_num_levs+1); + m_tmp_interface_view = view_2d("tmp_int",m_num_cols,npacks_p1); + } + } +} +// ========================================================================================= +void VerticalLayerDiagnostic::compute_diagnostic_impl() +{ + const auto npacks = ekat::npack(m_num_levs); + const auto default_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(m_num_cols, npacks); + + const auto& T_mid = get_field_in("T_mid").get_view(); + const auto& p_mid = get_field_in("p_mid").get_view(); + const auto& qv_mid = get_field_in("qv").get_view(); + const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); + + view_1d_const phis; + if (not m_only_compute_dz and not m_from_sea_level) { + phis = get_field_in("phis").get_view(); + } + + const bool only_compute_dz = m_only_compute_dz; + const bool is_interface_layout = m_is_interface_layout; + const bool from_sea_level = m_from_sea_level; + const int num_levs = m_num_levs; + + // Alias correct view for diagnostic output and for tmp class views + view_2d interface_view; + view_2d midpoint_view; + if (only_compute_dz) { + midpoint_view = m_diagnostic_output.get_view(); + } else if (is_interface_layout) { + interface_view = m_diagnostic_output.get_view(); + midpoint_view = m_tmp_midpoint_view; + } else { + midpoint_view = m_diagnostic_output.get_view(); + interface_view = m_tmp_interface_view; + } + + Kokkos::parallel_for("VerticalLayerDiagnostic", + default_policy, + KOKKOS_LAMBDA(const MemberType& team) { + const int icol = team.league_rank(); + + // Calculate dz. Use the memory in tmp_mid_view for dz and z_mid, + // since we don't set z_mid until after dz is no longer needed. + const auto& dz_s = ekat::subview(midpoint_view, icol); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, npacks), [&] (const Int& jpack) { + dz_s(jpack) = PF::calculate_dz(pseudo_density_mid(icol,jpack), p_mid(icol,jpack), T_mid(icol,jpack), qv_mid(icol,jpack)); + }); + team.team_barrier(); + + if (not only_compute_dz) { + // Calculate z_int if this diagnostic is not dz + const auto& z_int_s = ekat::subview(interface_view, icol); + const Real surf_geopotential = from_sea_level ? 0.0 : phis(icol); + PF::calculate_z_int(team,num_levs,dz_s,surf_geopotential,z_int_s); + + if (not is_interface_layout) { + // Calculate z_mid if this diagnostic is not dz or an interface value + const auto& z_mid_s = ekat::subview(midpoint_view, icol); + PF::calculate_z_mid(team,num_levs,z_int_s,z_mid_s); + } + } + }); +} +// ========================================================================================= +} //namespace scream diff --git a/components/eamxx/src/diagnostics/vertical_layer.hpp b/components/eamxx/src/diagnostics/vertical_layer.hpp new file mode 100644 index 000000000000..10aefff9d841 --- /dev/null +++ b/components/eamxx/src/diagnostics/vertical_layer.hpp @@ -0,0 +1,77 @@ +#ifndef EAMXX_VERTICAL_LAY_MID_DIAGNOSTIC_HPP +#define EAMXX_VERTICAL_LAY_MID_DIAGNOSTIC_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/util/scream_common_physics_functions.hpp" +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream +{ + +/* + * This diagnostic will produce data related to the vertical layer based on + * the parameter "diag_name" (required). The following can be produced: + * - diag_name = "dz": Vertical layer thickness for each column and level + * - diag_name = "z_int": Vertical layer height at each column and interface level. + * Values are computed from sea level (i.e., surf_geopotential=0). + * - diag_name = "geopotential_int": Same as z_int, but computed from topography data + * (i.e., surf_geopotential=phis). + * - diag_name = "z_mid"/"geopotential_mid": Same as z_int/geopotential_int but at midpoint levels. + */ + +class VerticalLayerDiagnostic : public AtmosphereDiagnostic +{ +public: + using Pack = ekat::Pack; + using PF = scream::PhysicsFunctions; + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + using view_1d_const = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; + + // Constructors + VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); + + // Set type to diagnostic + AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } + + // The name of the diagnostic. + std::string name () const { return m_diag_name; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + +protected: +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void compute_diagnostic_impl (); +protected: + + // Keep track of field dimensions + Int m_num_cols; + Int m_num_levs; + + // Temporary view to set dz, z_mid, and z_int + view_2d m_tmp_interface_view; + view_2d m_tmp_midpoint_view; + + // The diagnostic name. This will dictate which + // field in the computation is output (dz, z_int, or z_mid). + std::string m_diag_name; + + // Store if we only need to compute dz to save computation/memory requirements. + bool m_only_compute_dz; + + // Store if the diagnostic output field exists on interface values + bool m_is_interface_layout; + + // If z_int or z_mid is computed, determine whether the BC + // is from sea level or not (from topography data). + bool m_from_sea_level; + +}; // class VerticalLayerDiagnostic + +} //namespace scream + +#endif // EAMXX_VERTICAL_LAY_MID_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/vertical_layer_interface.cpp b/components/eamxx/src/diagnostics/vertical_layer_interface.cpp deleted file mode 100644 index 229be0cfd525..000000000000 --- a/components/eamxx/src/diagnostics/vertical_layer_interface.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include "diagnostics/vertical_layer_interface.hpp" - -namespace scream -{ - -// ========================================================================================= -VerticalLayerInterfaceDiagnostic::VerticalLayerInterfaceDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void VerticalLayerInterfaceDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar3d_layout_int { {COL,ILEV}, {m_num_cols,m_num_levs+1} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - - // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar3d_layout_int, m, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(ps); - m_diagnostic_output.allocate_view(); - - // Initialize a 2d view of dz to be used for compute_diagnostic - const auto npacks = ekat::npack(m_num_levs); - m_dz = view_2d("",m_num_cols,npacks); -} -// ========================================================================================= -void VerticalLayerInterfaceDiagnostic::compute_diagnostic_impl() -{ - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(m_num_cols, npacks); - const auto& z_int = m_diagnostic_output.get_view(); - const auto& T_mid = get_field_in("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& qv_mid = get_field_in("qv").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - - // Set surface geopotential for this diagnostic - const Real surf_geopotential = 0.0; - - const int num_levs = m_num_levs; - auto dz = m_dz; - Kokkos::parallel_for("VerticalLayerInterfaceDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - const auto& dz_s = ekat::subview(dz, icol); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, npacks), [&] (const Int& jpack) { - dz_s(jpack) = PF::calculate_dz(pseudo_density_mid(icol,jpack), p_mid(icol,jpack), T_mid(icol,jpack), qv_mid(icol,jpack)); - }); - team.team_barrier(); - const auto& z_int_s = ekat::subview(z_int, icol); - PF::calculate_z_int(team,num_levs,dz_s,surf_geopotential,z_int_s); - }); - - const auto ts = get_field_in("qv").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/vertical_layer_interface.hpp b/components/eamxx/src/diagnostics/vertical_layer_interface.hpp deleted file mode 100644 index ab9b1d9a9423..000000000000 --- a/components/eamxx/src/diagnostics/vertical_layer_interface.hpp +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef EAMXX_VERTICAL_LAY_INT_DIAGNOSTIC_HPP -#define EAMXX_VERTICAL_LAY_INT_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the potential temperature. - */ - -class VerticalLayerInterfaceDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - - using view_2d = typename KT::template view_2d; - - // Constructors - VerticalLayerInterfaceDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "VerticalLayerInterface"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - - // Temporary view to set dz in compute diagnostic - view_2d m_dz; - -}; // class VerticalLayerInterfaceDiagnostic - -} //namespace scream - -#endif // EAMXX_VERTICAL_LAY_INT_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/vertical_layer_midpoint.cpp b/components/eamxx/src/diagnostics/vertical_layer_midpoint.cpp deleted file mode 100644 index a8293b94eba6..000000000000 --- a/components/eamxx/src/diagnostics/vertical_layer_midpoint.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include "diagnostics/vertical_layer_midpoint.hpp" - -namespace scream -{ - -// ========================================================================================= -VerticalLayerMidpointDiagnostic:: -VerticalLayerMidpointDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void VerticalLayerMidpointDiagnostic:: -set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - - // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar3d_layout_mid, m, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(ps); - m_diagnostic_output.allocate_view(); - - // Initialize 2d view of dz to be used in compute_diagnostic - const auto npacks_p1 = ekat::npack(m_num_levs+1); - m_z_int = view_2d("",m_num_cols,npacks_p1); -} -// ========================================================================================= -void VerticalLayerMidpointDiagnostic::compute_diagnostic_impl() -{ - - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(m_num_cols, npacks); - const auto& z_mid = m_diagnostic_output.get_view(); - const auto& T_mid = get_field_in("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& qv_mid = get_field_in("qv").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - - // Set surface geopotential for this diagnostic - const Real surf_geopotential = 0.0; - - const int num_levs = m_num_levs; - auto z_int = m_z_int; - Kokkos::parallel_for("VerticalLayerMidpointDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - const auto& z_mid_s = ekat::subview(z_mid, icol); - const auto& dz_s = z_mid_s; // Use the memory in z_mid for dz, since we don't set z_mid until after dz is no longer needed. - const auto& z_int_s = ekat::subview(z_int, icol); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, npacks), [&] (const Int& jpack) { - dz_s(jpack) = PF::calculate_dz(pseudo_density_mid(icol,jpack), p_mid(icol,jpack), T_mid(icol,jpack), qv_mid(icol,jpack)); - }); - team.team_barrier(); - PF::calculate_z_int(team,num_levs,dz_s,surf_geopotential,z_int_s); - PF::calculate_z_mid(team,num_levs,z_int_s,z_mid_s); - }); - - const auto ts = get_field_in("qv").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/vertical_layer_midpoint.hpp b/components/eamxx/src/diagnostics/vertical_layer_midpoint.hpp deleted file mode 100644 index 979b7b330a35..000000000000 --- a/components/eamxx/src/diagnostics/vertical_layer_midpoint.hpp +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef EAMXX_VERTICAL_LAY_MID_DIAGNOSTIC_HPP -#define EAMXX_VERTICAL_LAY_MID_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the potential temperature. - */ - -class VerticalLayerMidpointDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_2d = typename KT::template view_2d; - - // Constructors - VerticalLayerMidpointDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "VerticalLayerMidpoint"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - - // Temporary view to set dz in compute diagnostic - view_2d m_z_int; - -}; // class VerticalLayerMidpointDiagnostic - -} //namespace scream - -#endif // EAMXX_VERTICAL_LAY_MID_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/vertical_layer_thickness.cpp b/components/eamxx/src/diagnostics/vertical_layer_thickness.cpp deleted file mode 100644 index 1f83823c22a1..000000000000 --- a/components/eamxx/src/diagnostics/vertical_layer_thickness.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include "diagnostics/vertical_layer_thickness.hpp" - -namespace scream -{ - -// ========================================================================================= -VerticalLayerThicknessDiagnostic::VerticalLayerThicknessDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void VerticalLayerThicknessDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - - // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar3d_layout_mid, m, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(ps); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void VerticalLayerThicknessDiagnostic::compute_diagnostic_impl() -{ - - const auto npacks = ekat::npack(m_num_levs); - const auto& dz = m_diagnostic_output.get_view(); - const auto& T_mid = get_field_in("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& qv_mid = get_field_in("qv").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - - - Kokkos::parallel_for("VerticalLayerThicknessDiagnostic", - Kokkos::RangePolicy<>(0,m_num_cols*npacks), - KOKKOS_LAMBDA(const int& idx) { - const int icol = idx / npacks; - const int jpack = idx % npacks; - dz(icol,jpack) = PF::calculate_dz(pseudo_density_mid(icol,jpack), p_mid(icol,jpack), T_mid(icol,jpack), qv_mid(icol,jpack)); - }); - Kokkos::fence(); - - const auto ts = get_field_in("qv").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/vertical_layer_thickness.hpp b/components/eamxx/src/diagnostics/vertical_layer_thickness.hpp deleted file mode 100644 index 4e2913ef42f1..000000000000 --- a/components/eamxx/src/diagnostics/vertical_layer_thickness.hpp +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef EAMXX_VERTICAL_LAY_THICK_DIAGNOSTIC_HPP -#define EAMXX_VERTICAL_LAY_THICK_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the potential temperature. - */ - -class VerticalLayerThicknessDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - - // Constructors - VerticalLayerThicknessDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "VerticalLayerThickness"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - -}; // class VerticalLayerThicknessDiagnostic - -} //namespace scream - -#endif // EAMXX_VERTICAL_LAY_THICK_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/virtual_temperature.cpp b/components/eamxx/src/diagnostics/virtual_temperature.cpp index b208960395c9..9ddcd18549bb 100644 --- a/components/eamxx/src/diagnostics/virtual_temperature.cpp +++ b/components/eamxx/src/diagnostics/virtual_temperature.cpp @@ -3,14 +3,13 @@ namespace scream { -// ========================================================================================= -VirtualTemperatureDiagnostic::VirtualTemperatureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) +VirtualTemperatureDiagnostic:: +VirtualTemperatureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) { // Nothing to do here } -// ========================================================================================= void VirtualTemperatureDiagnostic::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -29,7 +28,7 @@ void VirtualTemperatureDiagnostic::set_grids(const std::shared_ptr("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); + add_field("qv", scalar3d_layout_mid, Q, grid_name, ps); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, K, grid_name); @@ -38,10 +37,9 @@ void VirtualTemperatureDiagnostic::set_grids(const std::shared_ptr(m_num_levs); const auto& virtualT = m_diagnostic_output.get_view(); const auto& T_mid = get_field_in("T_mid").get_view(); @@ -55,9 +53,6 @@ void VirtualTemperatureDiagnostic::compute_diagnostic_impl() virtualT(icol,jpack) = PF::calculate_virtual_temperature(T_mid(icol,jpack),qv_mid(icol,jpack)); }); Kokkos::fence(); - - const auto ts = get_field_in("qv").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); } -// ========================================================================================= + } //namespace scream diff --git a/components/eamxx/src/diagnostics/water_path.cpp b/components/eamxx/src/diagnostics/water_path.cpp new file mode 100644 index 000000000000..2e8da274bfbe --- /dev/null +++ b/components/eamxx/src/diagnostics/water_path.cpp @@ -0,0 +1,95 @@ +#include "diagnostics/water_path.hpp" +#include "physics/share/physics_constants.hpp" + +#include + +namespace scream +{ + +WaterPathDiagnostic:: +WaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) +{ + EKAT_REQUIRE_MSG (params.isParameter("Water Kind"), + "Error! WaterPathDiagnostic requires 'Water Kind' in its input parameters.\n"); + + m_kind = m_params.get("Water Kind"); + if (m_kind=="Liq") { + m_qname = "qc"; + } else if (m_kind=="Ice") { + m_qname = "qi"; + } else if (m_kind=="Rain") { + m_qname = "qr"; + } else if (m_kind=="Rime") { + m_qname = "qm"; + } else if (m_kind=="Vap") { + m_qname = "qv"; + } else { + EKAT_ERROR_MSG ( + "Error! Invalid choice for 'WaterKind' in WaterPathDiagnostic.\n" + " - input value: " + m_kind + "\n" + " - valid values: Liq, Ice, Rain, Rime, Vap\n"); + } +} + +std::string WaterPathDiagnostic::name() const +{ + return m_kind + "WaterPath"; +} + +void WaterPathDiagnostic:: +set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + + const auto m2 = m*m; + auto Q = kg/kg; + Q.set_string("kg/kg"); + + auto grid = grids_manager->get_grid("Physics"); + const auto& grid_name = grid->name(); + m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column + + auto scalar2d = grid->get_2d_scalar_layout(); + auto scalar3d = grid->get_3d_scalar_layout(true); + + // The fields required for this diagnostic to be computed + add_field("pseudo_density", scalar3d, Pa, grid_name); + add_field(m_qname, scalar3d, Q, grid_name); + + // Construct and allocate the diagnostic field + FieldIdentifier fid (name(), scalar2d, kg/m2, grid_name); + m_diagnostic_output = Field(fid); + m_diagnostic_output.allocate_view(); +} + +void WaterPathDiagnostic::compute_diagnostic_impl() +{ + using PC = scream::physics::Constants; + using KT = KokkosTypes; + using MT = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + + constexpr Real g = PC::gravit; + + const auto wp = m_diagnostic_output.get_view(); + const auto q = get_field_in(m_qname).get_view(); + const auto rho = get_field_in("pseudo_density").get_view(); + + const auto num_levs = m_num_levs; + const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); + Kokkos::parallel_for("Compute " + name(), policy, + KOKKOS_LAMBDA(const MT& team) { + const int icol = team.league_rank(); + auto q_icol = ekat::subview(q,icol); + auto rho_icol = ekat::subview(rho,icol); + Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), + [&] (const int& ilev, Real& lsum) { + lsum += q_icol(ilev) * rho_icol(ilev) / g; + },wp(icol)); + team.team_barrier(); + }); +} + +} //namespace scream diff --git a/components/eamxx/src/diagnostics/water_path.hpp b/components/eamxx/src/diagnostics/water_path.hpp new file mode 100644 index 000000000000..271632fcef68 --- /dev/null +++ b/components/eamxx/src/diagnostics/water_path.hpp @@ -0,0 +1,45 @@ +#ifndef EAMXX_ggWATER_PATH_DIAGNOSTIC_HPP +#define EAMXX_ggWATER_PATH_DIAGNOSTIC_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream +{ + +/* + * This diagnostic will produce the potential temperature. + */ + +class WaterPathDiagnostic : public AtmosphereDiagnostic +{ +public: + // Constructors + WaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); + + // Set type to diagnostic + AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } + + // The name of the diagnostic + std::string name () const; + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + +protected: +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void compute_diagnostic_impl (); +protected: + + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + std::string m_qname; + std::string m_kind; +}; // class WaterPathDiagnostic + +} //namespace scream + +#endif // EAMXX_ggWATER_PATH_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/zonal_vapor_flux.cpp b/components/eamxx/src/diagnostics/zonal_vapor_flux.cpp deleted file mode 100644 index e9d6979e058f..000000000000 --- a/components/eamxx/src/diagnostics/zonal_vapor_flux.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include "diagnostics/zonal_vapor_flux.hpp" - -namespace scream -{ - -// ========================================================================================= -ZonalVapFluxDiagnostic::ZonalVapFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereDiagnostic(comm,params) -{ - // Nothing to do here -} - -// ========================================================================================= -void ZonalVapFluxDiagnostic::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto vel = m/s; - vel.set_string("m/s"); - - auto grid = grids_manager->get_grid("Physics"); - const auto& grid_name = grid->name(); - m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar2d_layout_mid { {COL}, {m_num_cols} }; - FieldLayout horiz_wind_layout { {COL,CMP,LEV}, {m_num_cols,2,m_num_levs} }; - constexpr int ps = Pack::n; - - // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - // Note both u and v are packaged into the single field horiz_winds. - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - - - // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d_layout_mid, kg/m/s, grid_name); - m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(); - m_diagnostic_output.allocate_view(); -} -// ========================================================================================= -void ZonalVapFluxDiagnostic::compute_diagnostic_impl() -{ - - using PC = scream::physics::Constants; - constexpr Real gravit = PC::gravit; - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, npacks); - const auto& qv_vert_integrated_flux_u = m_diagnostic_output.get_view(); - const auto& qv_mid = get_field_in("qv").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); - const auto& horiz_winds = get_field_in("horiz_winds").get_view(); - - const auto num_levs = m_num_levs; - Kokkos::parallel_for("ZonalVapFluxDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const Int& idx, Real& lsum) { - const int jpack = idx / Pack::n; - const int klev = idx % Pack::n; - // Note, horiz_winds contains u (index 0) and v (index 1). Here we want u - lsum += horiz_winds(icol,0,jpack)[klev] * qv_mid(icol,jpack)[klev] * pseudo_density_mid(icol,jpack)[klev]/gravit; - },qv_vert_integrated_flux_u(icol)); - team.team_barrier(); - }); - - const auto ts = get_field_in("qv").get_header().get_tracking().get_time_stamp(); - m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} -// ========================================================================================= -} //namespace scream diff --git a/components/eamxx/src/diagnostics/zonal_vapor_flux.hpp b/components/eamxx/src/diagnostics/zonal_vapor_flux.hpp deleted file mode 100644 index b1a56dc15abe..000000000000 --- a/components/eamxx/src/diagnostics/zonal_vapor_flux.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef EAMXX_ZONAL_VAPOR_FLUX_DIAGNOSTIC_HPP -#define EAMXX_ZONAL_VAPOR_FLUX_DIAGNOSTIC_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream -{ - -/* - * This diagnostic will produce the zonal water vapor flux. - */ - -class ZonalVapFluxDiagnostic : public AtmosphereDiagnostic -{ -public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - - // Constructors - ZonalVapFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - - // The name of the diagnostic - std::string name () const { return "ZonalVaporFlux"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - -protected: -#ifdef KOKKOS_ENABLE_CUDA -public: -#endif - void compute_diagnostic_impl (); -protected: - - // Keep track of field dimensions - Int m_num_cols; - Int m_num_levs; - -}; // class VapWaterPathDiagnostic - -} //namespace scream - -#endif // EAMXX_ZONAL_VAPOR_FLUX_HPP diff --git a/components/eamxx/src/doubly-periodic/CMakeLists.txt b/components/eamxx/src/doubly-periodic/CMakeLists.txt new file mode 100644 index 000000000000..09dab68a8e28 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/CMakeLists.txt @@ -0,0 +1,60 @@ +include (ScreamUtils) + +set(DP_SRCS + dp_f90.cpp + dp_iso_c.f90 + #${SCREAM_BASE_DIR}/../eam/src/control/apply_iop_forcing.F90 + #${SCREAM_BASE_DIR}/../eam/src/dynamics/se/se_iop_intr_mod.F90", + #${SCREAM_BASE_DIR}/../eam/src/control/iop_data_mod.F90", + #${SCREAM_BASE_DIR}/../eam/src/control/history_iop.F90" +) + +# Set cmake config options for Homme +if (NOT "${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") + message(FATAL_ERROR "Requires homme") +endif() + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 10) + +if (NOT SCREAM_LIB_ONLY) + list(APPEND DP_SRCS + dp_functions_f90.cpp + ) # Add f90 bridges needed for testing +endif() + +# Add ETI source files if not on CUDA/HIP +if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) + list(APPEND DP_SRCS + eti/dp_advance_iop_forcing.cpp + eti/dp_advance_iop_nudging.cpp + eti/dp_advance_iop_subsidence.cpp + eti/dp_iop_setinitial.cpp + eti/dp_iop_broadcast.cpp + eti/dp_apply_iop_forcing.cpp + eti/dp_iop_domain_relaxation.cpp + eti/dp_crm_resolved_turb.cpp + eti/dp_iop_default_opts.cpp + eti/dp_iop_setopts.cpp + eti/dp_setiopupdate_init.cpp + eti/dp_setiopupdate.cpp + eti/dp_readiopdata.cpp + eti/dp_iop_intht.cpp + ) # DP ETI SRCS +endif() + +add_library(dp ${DP_SRCS}) +set_target_properties(dp PROPERTIES + Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules +) +target_include_directories(dp PUBLIC + ${CMAKE_CURRENT_BINARY_DIR}/modules + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/impl +) +target_link_libraries(dp PUBLIC physics_share scream_share ${dynLibName}) + +#if (NOT SCREAM_LIB_ONLY) +# add_subdirectory(tests) +#endif() diff --git a/components/eamxx/src/doubly-periodic/dp_constants.hpp b/components/eamxx/src/doubly-periodic/dp_constants.hpp new file mode 100644 index 000000000000..c974106f83ec --- /dev/null +++ b/components/eamxx/src/doubly-periodic/dp_constants.hpp @@ -0,0 +1,22 @@ +#ifndef DP_CONSTANTS_HPP +#define DP_CONSTANTS_HPP + +namespace scream { +namespace dp { + +/* + * Mathematical constants used by dp. + */ + +template +struct Constants +{ + static constexpr Scalar iop_nudge_tq_low = 1050; + static constexpr Scalar iop_nudge_tq_high = 0; + static constexpr Scalar iop_nudge_tscale = 10800; +}; + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/dp_f90.cpp b/components/eamxx/src/doubly-periodic/dp_f90.cpp new file mode 100644 index 000000000000..2802e5266fd5 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/dp_f90.cpp @@ -0,0 +1,27 @@ +#include "dp_f90.hpp" +#include "physics_constants.hpp" + +#include "ekat/ekat_assert.hpp" + +using scream::Int; + +extern "C" { + +void init_time_level_c (const int& nm1, const int& n0, const int& np1, + const int& nstep, const int& nstep0); + +} + +namespace scream { +namespace dp { + +void dp_init(const bool force_reinit) { + static bool is_init = false; + if (!is_init || force_reinit) { + init_time_level_c(10, 3, 11, 5, 4); + is_init = true; + } +} + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/dp_f90.hpp b/components/eamxx/src/doubly-periodic/dp_f90.hpp new file mode 100644 index 000000000000..338a583f777a --- /dev/null +++ b/components/eamxx/src/doubly-periodic/dp_f90.hpp @@ -0,0 +1,18 @@ +#ifndef SCREAM_DP_F90_HPP +#define SCREAM_DP_F90_HPP + +#include "share/scream_types.hpp" + +#include +#include + +namespace scream { +namespace dp { + +// Initialize DP. This is only for standalone DP testing. +void dp_init(const bool force_reinit=false); + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/dp_functions.hpp b/components/eamxx/src/doubly-periodic/dp_functions.hpp new file mode 100644 index 000000000000..18e6ec58ba9e --- /dev/null +++ b/components/eamxx/src/doubly-periodic/dp_functions.hpp @@ -0,0 +1,325 @@ +#ifndef DP_FUNCTIONS_HPP +#define DP_FUNCTIONS_HPP + +#include "physics/share/physics_constants.hpp" +#include "dp_constants.hpp" + +#include "share/scream_types.hpp" + +#include "ekat/ekat_pack_kokkos.hpp" +#include "ekat/ekat_workspace.hpp" + +#include "Elements.hpp" +#include "Tracers.hpp" + +namespace scream { +namespace dp { + +/* + * Functions is a stateless struct used to encapsulate a + * number of functions for DP. We use the ETI pattern for + * these functions. + * + * DP assumptions: + * - Kokkos team policies have a vector length of 1 + */ + +using element_t = Homme::Elements; +using tracer_t = Homme::Tracers; +struct hvcoord_t{}; +struct timelevel_t{}; +struct hybrid_t{}; + +template +struct Functions +{ + // + // ------- Types -------- + // + + using Scalar = ScalarT; + using Device = DeviceT; + + template + using BigPack = ekat::Pack; + template + using SmallPack = ekat::Pack; + + using IntSmallPack = SmallPack; + using Pack = BigPack; + using Spack = SmallPack; + + using Mask = ekat::Mask; + using Smask = ekat::Mask; + + using KT = ekat::KokkosTypes; + using ExeSpace = typename KT::ExeSpace; + + using C = physics::Constants; + using DPC = dp::Constants; + + template + using view_1d = typename KT::template view_1d; + template + using view_2d = typename KT::template view_2d; + template + using view_3d = typename KT::template view_3d; + + template + using view_1d_ptr_array = typename KT::template view_1d_ptr_carray; + + template + using uview_1d = typename ekat::template Unmanaged >; + + template + using uview_2d = typename ekat::template Unmanaged >; + + using MemberType = typename KT::MemberType; + + using WorkspaceMgr = typename ekat::WorkspaceManager; + using Workspace = typename WorkspaceMgr::Workspace; + + // + // --------- Functions --------- + // + + // --------------------------------------------------------------------- + // Define the pressures of the interfaces and midpoints from the + // coordinate definitions and the surface pressure. + // --------------------------------------------------------------------- + KOKKOS_FUNCTION + static void plevs0( + // Input arguments + const Int& nver, // vertical dimension + const Scalar& ps, // Surface pressure (pascals) + const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces + const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints + const uview_1d& hybi, // ps component of hybrid coordinate - interfaces + const uview_1d& hybm, // ps component of hybrid coordinate - midpoints + // Kokkos stuff + const MemberType& team, + // Output arguments + const uview_1d& pint, // Pressure at model interfaces + const uview_1d& pmid, // Pressure at model levels + const uview_1d& pdel); // Layer thickness (pint(k+1) - pint(k)) + + //----------------------------------------------------------------------- + // advance_iop_forcing + // Purpose: + // Apply large scale forcing for t, q, u, and v as provided by the + // case IOP forcing file. + // + // Author: + // Original version: Adopted from CAM3.5/CAM5 + // Updated version for E3SM: Peter Bogenschutz (bogenschutz1@llnl.gov) + // and replaces the forecast.F90 routine in CAM3.5/CAM5/CAM6/E3SMv1/E3SMv2 + // CXX version: James Foucar (jgfouca@sandia.gov) + // + //----------------------------------------------------------------------- + KOKKOS_FUNCTION + static void advance_iop_forcing( + // Input arguments + const Int& plev, // number of vertical levels + const Int& pcnst, // number of advected constituents including cloud water + const bool& have_u, // dataset contains u + const bool& have_v, // dataset contains v + const bool& dp_crm, // use 3d forcing + const bool& use_3dfrc, // use 3d forcing + const Scalar& scm_dt, // model time step [s] + const Scalar& ps_in, // surface pressure [Pa] + const uview_1d& u_in, // zonal wind [m/s] + const uview_1d& v_in, // meridional wind [m/s] + const uview_1d& t_in, // temperature [K] + const uview_2d& q_in, // q tracer array [units vary] + const uview_1d& t_phys_frc, // temperature forcing from physics [K/s] + const uview_1d& divt3d, // 3D T advection + const uview_2d& divq3d, // 3D q advection + const uview_1d& divt, // Divergence of temperature + const uview_2d& divq, // Divergence of moisture + const uview_1d& wfld, // Vertical motion (slt) + const uview_1d& uobs, // actual u wind + const uview_1d& vobs, // actual v wind + const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces + const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints + const uview_1d& hybi, // ps component of hybrid coordinate - interfaces + const uview_1d& hybm, // ps component of hybrid coordinate - midpoints + // Kokkos stuff + const MemberType& team, + const Workspace& workspace, + // Output arguments + const uview_1d& u_update, // updated temperature [K] + const uview_1d& v_update, // updated q tracer array [units vary] + const uview_1d& t_update, // updated zonal wind [m/s] + const uview_2d& q_update); // updated meridional wind [m/s] + + //----------------------------------------------------------------------- + // advance_iop_nudging + // Purpose: + // Option to nudge t and q to observations as specified by the IOP file + // + // Author: + // Original version: Adopted from CAM3.5/CAM5 + // Updated version for E3SM: Peter Bogenschutz (bogenschutz1@llnl.gov) + // CXX version: Conrad Clevenger (tccleve@sandia.gov) + // + //----------------------------------------------------------------------- + KOKKOS_FUNCTION + static void advance_iop_nudging( + // Input arguments + const Int& plev, // number of vertical levels + const Scalar& scm_dt, // model time step [s] + const Scalar& ps_in, // surface pressure [Pa] + const uview_1d& t_in, // temperature [K] + const uview_1d& q_in, // water vapor mixing ratio [kg/kg] + const uview_1d& tobs, // observed temperature [K] + const uview_1d& qobs, // observed vapor mixing ratio [kg/kg] + const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces + const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints + const uview_1d& hybi, // ps component of hybrid coordinate - interfaces + const uview_1d& hybm, // ps component of hybrid coordinate - midpoints + // Kokkos stuff + const MemberType& team, + const Workspace& workspace, + // Output arguments + const uview_1d& t_update, // updated temperature [K] + const uview_1d& q_update, // updated water vapor [kg/kg] + const uview_1d& relaxt, // relaxation of temperature [K/s] + const uview_1d& relaxq); // relaxation of vapor [kg/kg/s] + + KOKKOS_INLINE_FUNCTION + static void do_advance_iop_subsidence_update( + const Int& k, + const Int& plev, + const Spack& fac, + const Spack& swfldint, + const Spack& swfldint_p1, + const uview_1d& in, + const uview_1d& in_s, + const uview_1d& update); + + //----------------------------------------------------------------------- + // + // Purpose: + // Option to compute effects of large scale subsidence on T, q, u, and v. + // Code originated from CAM3.5/CAM5 Eulerian subsidence computation for SCM + // in the old forecast.f90 routine. + //----------------------------------------------------------------------- + KOKKOS_FUNCTION + static void advance_iop_subsidence( + // Input arguments + const Int& plev, // number of vertical levels + const Int& pcnst, // number of advected constituents including cloud water + const Scalar& scm_dt, // model time step [s] + const Scalar& ps_in, // surface pressure [Pa] + const uview_1d& u_in, // zonal wind [m/s] + const uview_1d& v_in, // meridional wind [m/s] + const uview_1d& t_in, // temperature [K] + const uview_2d& q_in, // tracer [vary] + const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces + const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints + const uview_1d& hybi, // ps component of hybrid coordinate - interfaces + const uview_1d& hybm, // ps component of hybrid coordinate - midpoints + const uview_1d& wfld, // Vertical motion (slt) + // Kokkos stuff + const MemberType& team, + const Workspace& workspace, + // Output arguments + const uview_1d& u_update, // zonal wind [m/s] + const uview_1d& v_update, // meridional wind [m/s] + const uview_1d& t_update, // temperature [m/s] + const uview_2d& q_update); // tracer [vary] + + //--------------------------------------------------------- + // Purpose: Set initial values from IOP files (where available) + // when running SCM or DP-CRM. + //---------------------------------------------------------- + static void iop_setinitial( + // Input arguments + const Int& plev, // number of vertical levels + const Int& pcnst, // number of advected constituents including cloud water + const Int& nelemd, // number of elements per MPI task + const Int& np, // NP + const Int& nstep, // the timestep number + const bool& use_replay, // use e3sm generated forcing + const bool& dynproc, // Designation of a dynamics processor - AaronDonahue + const bool& have_t, // dataset contains t + const bool& have_q, // dataset contains q + const bool& have_ps, // dataset contains ps + const bool& have_u, // dataset contains u + const bool& have_v, // dataset contains v + const bool& have_numliq, // dataset contains numliq + const bool& have_cldliq, // dataset contains cldliq + const bool& have_numice, // dataset contains numice + const bool& have_cldice, // dataset contains cldice + const bool& scm_zero_non_iop_tracers, // Ignore all tracers from initial conditions file, and default all tracers not specified in IOP to minimum value (usually zero) + const bool& is_first_restart_step, // is first restart step + const uview_1d& qmin, // minimum permitted constituent concentration (kg/kg) (pcnst) + const uview_1d& uobs, // actual u wind + const uview_1d& vobs, // actual v wind + const uview_1d& numliqobs, // actual ??? + const uview_1d& numiceobs, // actual ??? + const uview_1d& cldliqobs, // actual ??? + const uview_1d& cldiceobs, // actual ??? + const Scalar& psobs, // ??? + const uview_1d& dx_short, // short length scale in km (nelemd) + // Input/Output arguments + Scalar& dyn_dx_size, // for use in doubly periodic CRM mode + tracer_t& tracers, // tracers + element_t& elem, // elements + const uview_1d& tobs, // actual temperature, dims=(plev) + const uview_1d& qobs); // actual W.V. Mixing ratio + + KOKKOS_FUNCTION + static void iop_broadcast(); + + KOKKOS_FUNCTION + static void apply_iop_forcing(const Int& nelemd, const uview_1d& elem, hvcoord_t& hvcoord, const hybrid_t& hybrid, const timelevel_t& tl, const Int& n, const bool& t_before_advance, const Int& nets, const Int& nete); + + KOKKOS_FUNCTION + static void iop_domain_relaxation(const Int& nelemd, const Int& np, const Int& nlev, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const uview_1d& dp, const Int& nelemd_todo, const Int& np_todo, const Spack& dt); + + KOKKOS_FUNCTION + static void crm_resolved_turb(const Int& nelemd, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const Int& nelemd_todo, const Int& np_todo); + + static void iop_default_opts(Spack& scmlat_out, Spack& scmlon_out, std::string& iopfile_out, bool& single_column_out, bool& scm_iop_srf_prop_out, bool& iop_nudge_tq_out, bool& iop_nudge_uv_out, Spack& iop_nudge_tq_low_out, Spack& iop_nudge_tq_high_out, Spack& iop_nudge_tscale_out, bool& scm_observed_aero_out, bool& iop_dosubsidence_out, bool& scm_multcols_out, bool& dp_crm_out, Spack& iop_perturb_high_out, bool& precip_off_out, bool& scm_zero_non_iop_tracers_out); + + static void iop_setopts(const Spack& scmlat_in, const Spack& scmlon_in, const std::string& iopfile_in, const bool& single_column_in, const bool& scm_iop_srf_prop_in, const bool& iop_nudge_tq_in, const bool& iop_nudge_uv_in, const Spack& iop_nudge_tq_low_in, const Spack& iop_nudge_tq_high_in, const Spack& iop_nudge_tscale_in, const bool& scm_observed_aero_in, const bool& iop_dosubsidence_in, const bool& scm_multcols_in, const bool& dp_crm_in, const Spack& iop_perturb_high_in, const bool& precip_off_in, const bool& scm_zero_non_iop_tracers_in); + + KOKKOS_FUNCTION + static void setiopupdate_init(); + + KOKKOS_FUNCTION + static void setiopupdate(); + + KOKKOS_FUNCTION + static void readiopdata(const Int& plev, const bool& iop_update_phase1, const uview_1d& hyam, const uview_1d& hybm); + + KOKKOS_FUNCTION + static void iop_intht(); +}; // struct Functions + +} // namespace dp +} // namespace scream + +// If a GPU build, without relocatable device code enabled, make all code available +// to the translation unit; otherwise, ETI is used. +#if defined(EAMXX_ENABLE_GPU) && !defined(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) \ + && !defined(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) + +# include "impl/dp_advance_iop_forcing_impl.hpp" +# include "impl/dp_advance_iop_nudging_impl.hpp" +# include "impl/dp_advance_iop_subsidence_impl.hpp" +# include "impl/dp_iop_setinitial_impl.hpp" +# include "impl/dp_iop_broadcast_impl.hpp" +# include "impl/dp_apply_iop_forcing_impl.hpp" +# include "impl/dp_iop_domain_relaxation_impl.hpp" +# include "impl/dp_crm_resolved_turb_impl.hpp" +# include "impl/dp_iop_default_opts_impl.hpp" +# include "impl/dp_iop_setopts_impl.hpp" +# include "impl/dp_setiopupdate_init_impl.hpp" +# include "impl/dp_setiopupdate_impl.hpp" +# include "impl/dp_readiopdata_impl.hpp" +# include "impl/dp_iop_intht_impl.hpp" +#endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE + +#endif // DP_FUNCTIONS_HPP diff --git a/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp b/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp new file mode 100644 index 000000000000..bad3c68dcb2d --- /dev/null +++ b/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp @@ -0,0 +1,526 @@ +#include "dp_functions_f90.hpp" + +#include "dp_f90.hpp" + +#include "ekat/ekat_assert.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "ekat/ekat_pack_kokkos.hpp" +#include "ekat/kokkos/ekat_subview_utils.hpp" + +#include "share/util/scream_deep_copy.hpp" + +#include + +using scream::Real; +using scream::Int; + +using scream::dp::element_t; +using scream::dp::hvcoord_t; +using scream::dp::hybrid_t; +using scream::dp::timelevel_t; + +// +// A C interface to DP fortran calls. The stubs below will link to fortran definitions in dp_iso_c.f90 +// + +extern "C" { + +void advance_iop_forcing_c(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* u_update, Real* v_update, Real* t_update, Real* q_update); +void advance_iop_nudging_c(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* t_update, Real* q_update, Real* relaxt, Real* relaxq); +void advance_iop_subsidence_c(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* u_update, Real* v_update, Real* t_update, Real* q_update); +void iop_setinitial_c(Int nelemd, element_t* elem); +void iop_broadcast_c(); +void apply_iop_forcing_c(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t* hybrid, timelevel_t* tl, Int n, bool t_before_advance, Int nets, Int nete); +void iop_domain_relaxation_c(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt); +void crm_resolved_turb_c(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo); +void iop_default_opts_c(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out); +void iop_setopts_c(Real scmlat_in, Real scmlon_in, const char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in); +void setiopupdate_init_c(); +void setiopupdate_c(); +void readiopdata_c(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm); +void iop_intht_c(); +} // extern "C" : end _c decls + +namespace scream { +namespace dp { + +// +// Glue functions to call fortran from from C++ with the Data struct +// + +void advance_iop_forcing(AdvanceIopForcingData& d) +{ + dp_init(); + d.transpose(); + advance_iop_forcing_c(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.t_phys_frc, d.u_update, d.v_update, d.t_update, d.q_update); + d.transpose(); +} + + +void advance_iop_nudging(AdvanceIopNudgingData& d) +{ + dp_init(); + advance_iop_nudging_c(d.plev, d.scm_dt, d.ps_in, d.t_in, d.q_in, d.t_update, d.q_update, d.relaxt, d.relaxq); +} + +void advance_iop_subsidence(AdvanceIopSubsidenceData& d) +{ + dp_init(); + d.transpose(); + advance_iop_subsidence_c(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.u_update, d.v_update, d.t_update, d.q_update); + d.transpose(); +} + +void iop_setinitial(IopSetinitialData& d) +{ + dp_init(); + //iop_setinitial_c(d.nelemd, d.elem); +} + +void iop_broadcast(IopBroadcastData& d) +{ + dp_init(); + iop_broadcast_c(); +} + +void apply_iop_forcing(ApplyIopForcingData& d) +{ + dp_init(); + apply_iop_forcing_c(d.nelemd, d.elem, &d.hvcoord, &d.hybrid, &d.tl, d.n, d.t_before_advance, d.nets, d.nete); +} + +void iop_domain_relaxation(IopDomainRelaxationData& d) +{ + dp_init(); + d.transpose(); + iop_domain_relaxation_c(d.nelemd, d.np, d.nlev, d.elem, d.hvcoord, d.hybrid, d.t1, d.dp, d.nelemd_todo, d.np_todo, d.dt); + d.transpose(); +} + +void crm_resolved_turb(CrmResolvedTurbData& d) +{ + dp_init(); + crm_resolved_turb_c(d.nelemd, d.elem, d.hvcoord, d.hybrid, d.t1, d.nelemd_todo, d.np_todo); +} + +void iop_default_opts(IopDefaultOptsData& d) +{ + dp_init(); + char cbuff[512] = ""; + char* buffptr = cbuff; + iop_default_opts_c(&d.scmlat_out, &d.scmlon_out, &buffptr, &d.single_column_out, &d.scm_iop_srf_prop_out, &d.iop_nudge_tq_out, &d.iop_nudge_uv_out, &d.iop_nudge_tq_low_out, &d.iop_nudge_tq_high_out, &d.iop_nudge_tscale_out, &d.scm_observed_aero_out, &d.iop_dosubsidence_out, &d.scm_multcols_out, &d.dp_crm_out, &d.iop_perturb_high_out, &d.precip_off_out, &d.scm_zero_non_iop_tracers_out); + d.iopfile_out = std::string(buffptr); +} + +void iop_setopts(IopSetoptsData& d) +{ + dp_init(); + const char* cptr = d.iopfile_in.c_str(); + iop_setopts_c(d.scmlat_in, d.scmlon_in, &cptr, d.single_column_in, d.scm_iop_srf_prop_in, d.iop_nudge_tq_in, d.iop_nudge_uv_in, d.iop_nudge_tq_low_in, d.iop_nudge_tq_high_in, d.iop_nudge_tscale_in, d.scm_observed_aero_in, d.iop_dosubsidence_in, d.scm_multcols_in, d.dp_crm_in, d.iop_perturb_high_in, d.precip_off_in, d.scm_zero_non_iop_tracers_in); +} + +void setiopupdate_init(SetiopupdateInitData& d) +{ + dp_init(); + setiopupdate_init_c(); +} + +void setiopupdate(SetiopupdateData& d) +{ + dp_init(); + setiopupdate_c(); +} + +void readiopdata(ReadiopdataData& d) +{ + dp_init(); + readiopdata_c(d.plev, d.iop_update_phase1, d.hyam, d.hybm); +} + +void iop_intht(IopInthtData& d) +{ + dp_init(); + iop_intht_c(); +} + +// end _c impls + +// +// _f function definitions. These expect data in C layout +// + +void advance_iop_forcing_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, bool have_u, bool have_v, bool dp_crm, bool use_3dfrc, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* divt3d, Real* divq3d, Real* divt, Real* divq, Real* wfld, Real* uobs, Real* vobs, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* u_update, Real* v_update, Real* t_update, Real* q_update) +{ + using DPF = Functions; + + using Spack = typename DPF::Spack; + using view_1d = typename DPF::view_1d; + using view_2d = typename DPF::view_2d; + using KT = typename DPF::KT; + using ExeSpace = typename KT::ExeSpace; + using MemberType = typename DPF::MemberType; + + // Some of the workspaces need plev+1 items + const Int plev_pack = ekat::npack(plev); + const Int plevp_pack = ekat::npack(plev+1); + + // Set up views + std::vector temp_d(AdvanceIopForcingData::NUM_ARRAYS-4); + std::vector temp_2d_d(4); + + ekat::host_to_device({u_in, v_in, t_in, t_phys_frc, divt3d, divt, wfld, uobs, vobs, hyai, hyam, hybi, hybm, u_update, v_update, t_update}, + plev, temp_d); + + ekat::host_to_device({ q_in, divq3d, divq, q_update }, + pcnst, plev, temp_2d_d, true); + + view_1d + u_in_d (temp_d[0]), + v_in_d (temp_d[1]), + t_in_d (temp_d[2]), + t_phys_frc_d (temp_d[3]), + divt3d_d (temp_d[4]), + divt_d (temp_d[5]), + wfld_d (temp_d[6]), + uobs_d (temp_d[7]), + vobs_d (temp_d[8]), + hyai_d (temp_d[9]), + hyam_d (temp_d[10]), + hybi_d (temp_d[11]), + hybm_d (temp_d[12]), + u_update_d (temp_d[13]), + v_update_d (temp_d[14]), + t_update_d (temp_d[15]); + + view_2d + q_in_d (temp_2d_d[0]), + divq3d_d (temp_2d_d[1]), + divq_d (temp_2d_d[2]), + q_update_d(temp_2d_d[3]); + + // Call core function from kernel + auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); + ekat::WorkspaceManager wsm(plevp_pack, 3, policy); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + + DPF::advance_iop_forcing( + plev, pcnst, have_u, have_v, dp_crm, use_3dfrc, scm_dt, ps_in, + u_in_d, v_in_d, t_in_d, q_in_d, t_phys_frc_d, divt3d_d, divq3d_d, divt_d, divq_d, wfld_d, uobs_d, vobs_d, hyai_d, hyam_d, hybi_d, hybm_d, + team, wsm.get_workspace(team), + u_update_d, v_update_d, t_update_d, q_update_d); + }); + + // Sync back to host + std::vector inout_views = {t_update_d, u_update_d, v_update_d}; + std::vector inout_views_2d = {q_update_d}; + + ekat::device_to_host({t_update, u_update, v_update}, plev, inout_views); + ekat::device_to_host({q_update}, pcnst, plev, inout_views_2d, true); +} + +void advance_iop_nudging_f(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* tobs, Real* qobs, + Real* hyai, Real* hyam, Real* hybi, Real* hybm, + Real* t_update, Real* q_update, Real* relaxt, Real* relaxq) +{ + using DPF = Functions; + + using Spack = typename DPF::Spack; + using view_1d = typename DPF::view_1d; + using KT = typename DPF::KT; + using ExeSpace = typename KT::ExeSpace; + using MemberType = typename DPF::MemberType; + + const Int plev_pack = ekat::npack(plev); + const Int plevp_pack = ekat::npack(plev+1); + + // Set up views + std::vector temp_d(AdvanceIopNudgingData::NUM_ARRAYS); + + ekat::host_to_device({t_in, q_in, tobs, qobs, hyai, hyam, hybi, hybm, t_update, q_update, relaxt, relaxq}, + plev, temp_d); + + int counter=0; + view_1d + t_in_d (temp_d[counter++]), + q_in_d (temp_d[counter++]), + tobs_d (temp_d[counter++]), + qobs_d (temp_d[counter++]), + hyai_d (temp_d[counter++]), + hyam_d (temp_d[counter++]), + hybi_d (temp_d[counter++]), + hybm_d (temp_d[counter++]), + t_update_d(temp_d[counter++]), + q_update_d(temp_d[counter++]), + relaxt_d (temp_d[counter++]), + relaxq_d (temp_d[counter++]); + + // Call core function from kernel + auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); + ekat::WorkspaceManager wsm(plevp_pack, 4, policy); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + + DPF::advance_iop_nudging( + plev, scm_dt, ps_in, t_in_d, q_in_d, tobs_d, qobs_d, + hyai_d, hyam_d, hybi_d, hybm_d, + team, wsm.get_workspace(team), + t_update_d, q_update_d, relaxt_d, relaxq_d); + }); + + // Sync back to host + std::vector out_views = {t_update_d, q_update_d, relaxt_d, relaxq_d}; + + ekat::device_to_host({t_update, q_update, relaxt, relaxq}, plev, out_views); +} + +void advance_iop_subsidence_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* wfld, Real* u_update, Real* v_update, Real* t_update, Real* q_update) +{ + using DPF = Functions; + + using Spack = typename DPF::Spack; + using view_1d = typename DPF::view_1d; + using view_2d = typename DPF::view_2d; + using KT = typename DPF::KT; + using ExeSpace = typename KT::ExeSpace; + using MemberType = typename DPF::MemberType; + + // Some of the workspaces need plev+1 items + const Int plev_pack = ekat::npack(plev); + const Int plevp_pack = ekat::npack(plev+1); + + // Set up views + std::vector temp_d(AdvanceIopSubsidenceData::NUM_ARRAYS-2); + std::vector temp_2d_d(2); + + ekat::host_to_device({u_in, v_in, t_in, hyai, hyam, hybi, hybm, wfld, u_update, v_update, t_update}, + plev, temp_d); + + ekat::host_to_device({ q_in, q_update }, + pcnst, plev, temp_2d_d, true); + + view_1d + u_in_d (temp_d[0]), + v_in_d (temp_d[1]), + t_in_d (temp_d[2]), + hyai_d (temp_d[3]), + hyam_d (temp_d[4]), + hybi_d (temp_d[5]), + hybm_d (temp_d[6]), + wfld_d (temp_d[7]), + u_update_d (temp_d[8]), + v_update_d (temp_d[9]), + t_update_d (temp_d[10]); + + view_2d + q_in_d (temp_2d_d[0]), + q_update_d(temp_2d_d[1]); + + // Call core function from kernel + auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); + ekat::WorkspaceManager wsm(plevp_pack, 4, policy); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + + DPF::advance_iop_subsidence( + plev, pcnst, scm_dt, ps_in, + u_in_d, v_in_d, t_in_d, q_in_d, hyai_d, hyam_d, hybi_d, hybm_d, wfld_d, + team, wsm.get_workspace(team), + u_update_d, v_update_d, t_update_d, q_update_d); + }); + + // Sync back to host + std::vector inout_views = {t_update_d, u_update_d, v_update_d}; + std::vector inout_views_2d = {q_update_d}; + + ekat::device_to_host({t_update, u_update, v_update}, plev, inout_views); + ekat::device_to_host({q_update}, pcnst, plev, inout_views_2d, true); +} + +void iop_setinitial_f(Int plev, Int pcnst, Int nelemd, Int np, Int nstep, Real psobs, bool use_replay, bool dynproc, bool have_t, bool have_q, bool have_ps, bool have_u, bool have_v, bool have_numliq, bool have_cldliq, bool have_numice, bool have_cldice, bool scm_zero_non_iop_tracers, bool is_first_restart_step, Real* qmin, Real* uobs, Real* vobs, Real* numliqobs, Real* numiceobs, Real* cldliqobs, Real* cldiceobs, Real* dx_short, tracer_t* tracers, element_t* elem, Real* dyn_dx_size, Real* tobs, Real* qobs) +{ + using DPF = Functions; + + using Spack = typename DPF::Spack; + using Scalarp = ekat::Pack; + using view_1d = typename DPF::view_1d; + using view_1ds = typename DPF::view_1d; + using sview_1ds = typename DPF::view_1d; + using KT = typename DPF::KT; + using ExeSpace = typename KT::ExeSpace; + using MemberType = typename DPF::MemberType; + + // Some of the workspaces need plev+1 items + const Int plev_pack = ekat::npack(plev); + + // Set up views + std::vector temp_d(8); + std::vector temp2_d(2); + + ekat::host_to_device({uobs, vobs, numliqobs, numiceobs, cldliqobs, cldiceobs, tobs, qobs}, + plev, temp_d); + + std::vector scalar_sizes = {nelemd, pcnst}; + ekat::host_to_device({dx_short, qmin}, scalar_sizes, temp2_d); + + view_1d + uobs_d (temp_d[0]), + vobs_d (temp_d[1]), + numliqobs_d (temp_d[2]), + numiceobs_d (temp_d[3]), + cldliqobs_d (temp_d[4]), + cldiceobs_d (temp_d[5]), + tobs_d (temp_d[6]), + qobs_d (temp_d[7]); + + sview_1ds + dx_short_d(reinterpret_cast(temp2_d[0].data()), scalar_sizes[0]), + qmin_d (reinterpret_cast(temp2_d[1].data()), scalar_sizes[1]); + + // Call core function + DPF::iop_setinitial(plev, pcnst, nelemd, np, nstep, use_replay, dynproc, have_t, have_ps, have_q, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step, qmin_d, uobs_d, vobs_d, numliqobs_d, numiceobs_d, cldliqobs_d, cldiceobs_d, psobs, dx_short_d, *dyn_dx_size, *tracers, *elem, tobs_d, qobs_d); + + // Sync back to host + std::vector inout_views = {tobs_d, qobs_d}; + + ekat::device_to_host({tobs, qobs}, plev, inout_views); +} + +void iop_broadcast_f() +{ +#if 0 + using PF = Functions; + + using Spack = typename PF::Spack; + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + PF::iop_broadcast(); + }); +#endif + +} +void apply_iop_forcing_f(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t hybrid, timelevel_t tl, Int n, bool t_before_advance, Int nets, Int nete) +{ + // TODO +} +void iop_domain_relaxation_f(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt) +{ + // TODO +} +void crm_resolved_turb_f(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo) +{ + // TODO +} +void iop_default_opts_f(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out) +{ +#if 0 + using PF = Functions; + + using Spack = typename PF::Spack; + using view_1d = typename PF::view_1d; + using bview_1d = typename PF::view_1d; + + view_1d t_d("t_d", 6); + const auto t_h = Kokkos::create_mirror_view(t_d); + + bview_1d bt_d("bt_d", 10); + const auto bt_h = Kokkos::create_mirror_view(bt_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack iop_nudge_tq_high_out_(), iop_nudge_tq_low_out_(), iop_nudge_tscale_out_(), iop_perturb_high_out_(), scmlat_out_(), scmlon_out_(); + bool dp_crm_out_(), iop_dosubsidence_out_(), iop_nudge_tq_out_(), iop_nudge_uv_out_(), precip_off_out_(), scm_iop_srf_prop_out_(), scm_multcols_out_(), scm_observed_aero_out_(), scm_zero_non_iop_tracers_out_(), single_column_out_(); + PF::iop_default_opts(scmlat_out_, scmlon_out_, iopfile_out_, single_column_out_, scm_iop_srf_prop_out_, iop_nudge_tq_out_, iop_nudge_uv_out_, iop_nudge_tq_low_out_, iop_nudge_tq_high_out_, iop_nudge_tscale_out_, scm_observed_aero_out_, iop_dosubsidence_out_, scm_multcols_out_, dp_crm_out_, iop_perturb_high_out_, precip_off_out_, scm_zero_non_iop_tracers_out_); + t_d(0) = iop_nudge_tq_high_out_[0]; + t_d(1) = iop_nudge_tq_low_out_[0]; + t_d(2) = iop_nudge_tscale_out_[0]; + t_d(3) = iop_perturb_high_out_[0]; + t_d(4) = scmlat_out_[0]; + t_d(5) = scmlon_out_[0]; + bt_d(0) = dp_crm_out_; + bt_d(1) = iop_dosubsidence_out_; + bt_d(2) = iop_nudge_tq_out_; + bt_d(3) = iop_nudge_uv_out_; + bt_d(4) = precip_off_out_; + bt_d(5) = scm_iop_srf_prop_out_; + bt_d(6) = scm_multcols_out_; + bt_d(7) = scm_observed_aero_out_; + bt_d(8) = scm_zero_non_iop_tracers_out_; + bt_d(9) = single_column_out_; + }); + Kokkos::deep_copy(t_h, t_d); + Kokkos::deep_copy(bt_h, bt_d); + *iop_nudge_tq_high_out = t_h(0); + *iop_nudge_tq_low_out = t_h(1); + *iop_nudge_tscale_out = t_h(2); + *iop_perturb_high_out = t_h(3); + *scmlat_out = t_h(4); + *scmlon_out = t_h(5); + *dp_crm_out = bt_h(0); + *iop_dosubsidence_out = bt_h(1); + *iop_nudge_tq_out = bt_h(2); + *iop_nudge_uv_out = bt_h(3); + *precip_off_out = bt_h(4); + *scm_iop_srf_prop_out = bt_h(5); + *scm_multcols_out = bt_h(6); + *scm_observed_aero_out = bt_h(7); + *scm_zero_non_iop_tracers_out = bt_h(8); + *single_column_out = bt_h(9); +#endif + +} +void iop_setopts_f(Real scmlat_in, Real scmlon_in, char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in) +{ +#if 0 + using PF = Functions; + + using Spack = typename PF::Spack; + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack iop_nudge_tq_high_in_(iop_nudge_tq_high_in), iop_nudge_tq_low_in_(iop_nudge_tq_low_in), iop_nudge_tscale_in_(iop_nudge_tscale_in), iop_perturb_high_in_(iop_perturb_high_in), scmlat_in_(scmlat_in), scmlon_in_(scmlon_in); + PF::iop_setopts(scmlat_in_, scmlon_in_, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in_, iop_nudge_tq_high_in_, iop_nudge_tscale_in_, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in_, precip_off_in, scm_zero_non_iop_tracers_in); + }); +#endif + +} +void setiopupdate_init_f() +{ +#if 0 + using PF = Functions; + + using Spack = typename PF::Spack; + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + PF::setiopupdate_init(); + }); +#endif + +} +void setiopupdate_f() +{ +#if 0 + using PF = Functions; + + using Spack = typename PF::Spack; + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + PF::setiopupdate(); + }); +#endif + +} +void readiopdata_f(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm) +{ + // TODO +} +void iop_intht_f() +{ +#if 0 + using PF = Functions; + + using Spack = typename PF::Spack; + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + PF::iop_intht(); + }); +#endif + +} +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp b/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp new file mode 100644 index 000000000000..055bfe4ad683 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp @@ -0,0 +1,306 @@ +#ifndef SCREAM_DP_FUNCTIONS_F90_HPP +#define SCREAM_DP_FUNCTIONS_F90_HPP + +#include "share/scream_types.hpp" +#include "physics/share/physics_test_data.hpp" + +#include "dp_functions.hpp" +#include "physics_constants.hpp" + +#include +#include +#include + +// +// Bridge functions to call fortran version of dp functions from C++ +// + +namespace scream { +namespace dp { + +struct AdvanceIopForcingData : public PhysicsTestData { + static constexpr size_t NUM_ARRAYS = 20; + + // Inputs + Int plev, pcnst; + Real scm_dt, ps_in; + bool have_u, have_v, dp_crm, use_3dfrc; + Real *u_in, *v_in, *t_in, *q_in, *t_phys_frc, *divt3d, *divq3d, *divt, + *divq, *wfld, *uobs, *vobs, *hyai, *hyam, *hybi, *hybm; + + // Outputs + Real *u_update, *v_update, *t_update, *q_update; + + AdvanceIopForcingData(Int plev_, Int pcnst_, Real scm_dt_, Real ps_in_, bool have_u_, bool have_v_, bool dp_crm_, bool use_3dfrc_) : + PhysicsTestData( + {{ plev_ }, { pcnst_, plev_ }}, + { + { &u_in, &v_in, &t_in, &t_phys_frc, &divt3d, &divt, &divq, &wfld, &uobs, &vobs, &hyai, &hyam, &hybi, &hybm, &u_update, &v_update, &t_update }, + { &q_in, &divq3d, &divq, &q_update } + }), + plev(plev_), pcnst(pcnst_), scm_dt(scm_dt_), ps_in(ps_in_), have_u(have_u_), have_v(have_v_), dp_crm(dp_crm_), use_3dfrc(use_3dfrc_) {} + + PTD_STD_DEF(AdvanceIopForcingData, 8, plev, pcnst, scm_dt, ps_in, have_u, have_v, dp_crm, use_3dfrc); +}; + + +struct AdvanceIopNudgingData : public PhysicsTestData { + static constexpr size_t NUM_ARRAYS = 12; + + // Inputs + Int plev; + Real scm_dt, ps_in; + Real *t_in, *q_in, *tobs, *qobs, *hyai, *hyam, *hybi, *hybm; + + // Outputs + Real *t_update, *q_update, *relaxt, *relaxq; + + AdvanceIopNudgingData(Int plev_, Real scm_dt_, Real ps_in_) : + PhysicsTestData({{ plev_ }}, {{ &t_in, &q_in, &tobs, &qobs, &hyai, &hyam, &hybi, &hybm, &t_update, &q_update, &relaxt, &relaxq }}), + plev(plev_), scm_dt(scm_dt_), ps_in(ps_in_) {} + + PTD_STD_DEF(AdvanceIopNudgingData, 3, plev, scm_dt, ps_in); +}; + +struct AdvanceIopSubsidenceData : public PhysicsTestData { + static constexpr size_t NUM_ARRAYS = 13; + + // Inputs + Int plev, pcnst; + Real scm_dt, ps_in; + Real *u_in, *v_in, *t_in, *q_in, *hyai, *hyam, *hybi, *hybm, *wfld; + + // Outputs + Real *u_update, *v_update, *t_update, *q_update; + + AdvanceIopSubsidenceData(Int plev_, Int pcnst_, Real scm_dt_, Real ps_in_) : + PhysicsTestData( + {{ plev_ }, { plev_, pcnst_ }}, + { + { &u_in, &v_in, &t_in, &hyai, &hyam, &hybi, &hybm, &wfld, &u_update, &v_update, &t_update }, + { &q_in, &q_update } + }), + plev(plev_), pcnst(pcnst_), scm_dt(scm_dt_), ps_in(ps_in_) {} + + PTD_STD_DEF(AdvanceIopSubsidenceData, 4, plev, pcnst, scm_dt, ps_in); +}; + +struct IopSetinitialData : public PhysicsTestData { + // Inputs + Int plev, pcnst, nelemd, np, nstep; + + bool use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step; + + Real psobs; + + Real* qmin, *uobs, *vobs, *numliqobs, *numiceobs, *cldliqobs, *cldiceobs, *dx_short; + + // Inputs/Outputs + tracer_t tracers; + element_t elem; + + Real dyn_dx_size; + + Real* tobs, *qobs; + + IopSetinitialData( + Int plev_, Int pcnst_, Int nelemd_, Int np_, Int nstep_, Real psobs_, + bool use_replay_, bool dynproc_, bool have_t_, bool have_q_, bool have_ps_, bool have_u_, bool have_v_, bool have_numliq_, bool have_cldliq_, bool have_numice_, bool have_cldice_, bool scm_zero_non_iop_tracers_, bool is_first_restart_step_) : + PhysicsTestData( + {{nelemd_}, {pcnst_}, {plev_}}, + { + {&dx_short}, + {&qmin}, + {&uobs, &vobs, &numliqobs, &numiceobs, &cldliqobs, &cldiceobs, &tobs, &qobs} + }), + plev(plev_), pcnst(pcnst_), nelemd(nelemd_), np(np_), nstep(nstep_), psobs(psobs_), + use_replay(use_replay_), dynproc(dynproc_), have_t(have_t_), have_q(have_q_), have_ps(have_ps_), have_u(have_u_), have_v(have_v_), have_numliq(have_numliq_), have_cldliq(have_cldliq_), have_numice(have_numice_), have_cldice(have_cldice_), scm_zero_non_iop_tracers(scm_zero_non_iop_tracers_), is_first_restart_step(is_first_restart_step_) { } + + PTD_STD_DEF(IopSetinitialData, 19, plev, pcnst, nelemd, np, nstep, psobs, use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step); + + void init() + { + tracers.init(nelemd, QSIZE_D); + elem.init(nelemd, true, true, 2); + } + + void randomize(std::mt19937_64& engine) + { + PhysicsTestData::randomize(engine); + init(); + + tracers.randomize(engine(), 0, 1); + elem.randomize(engine()); + + // Where tobs starts being non-zero is important to the algorithm + std::uniform_int_distribution default_int_dist(0, plev); + Int start_nz = default_int_dist(engine); + + for (Int k = 0; k < start_nz; ++k) { + tobs[k] = 0; + } + } +}; + +struct IopBroadcastData : public PhysicsTestData { + // Inputs + Int plev; + + IopBroadcastData(Int plev_=0) : + PhysicsTestData({}, {}), plev(plev_) {} + + PTD_STD_DEF(IopBroadcastData, 1, plev); +}; + +struct ApplyIopForcingData : public PhysicsTestData { + // Inputs + Int plev, nelemd, n, nets, nete; + hybrid_t hybrid; + timelevel_t tl; + bool t_before_advance; + + // Inputs/Outputs + element_t *elem; + hvcoord_t hvcoord; + + ApplyIopForcingData(Int plev_, Int nelemd_, Int n_, Int nets_, Int nete_, bool t_before_advance_) : + PhysicsTestData({}, {}), plev(plev_), nelemd(nelemd_), n(n_), nets(nets_), nete(nete_), t_before_advance(t_before_advance_) {} + + PTD_STD_DEF(ApplyIopForcingData, 6, plev, nelemd, n, nets, nete, t_before_advance); +}; + +struct IopDomainRelaxationData : public PhysicsTestData { + // Inputs + Int nelemd, np, nlev, t1, nelemd_todo, np_todo; + hvcoord_t hvcoord; + hybrid_t hybrid; + Real dt; + + // Inputs/Outputs + element_t *elem; + Real *dp; + + IopDomainRelaxationData(Int nelemd_, Int np_, Int nlev_, Int t1_, Int nelemd_todo_, Int np_todo_, Real dt_) : + PhysicsTestData({{ np_, np_, nlev_ }}, {{ &dp }}), nelemd(nelemd_), np(np_), nlev(nlev_), t1(t1_), nelemd_todo(nelemd_todo_), np_todo(np_todo_), dt(dt_) {} + + PTD_STD_DEF(IopDomainRelaxationData, 7, nelemd, np, nlev, t1, nelemd_todo, np_todo, dt); +}; + +struct CrmResolvedTurbData : public PhysicsTestData { + // Inputs + Int plev, nelemd, t1, nelemd_todo, np_todo; + hvcoord_t hvcoord; + hybrid_t hybrid; + + // Inputs/Outputs + element_t *elem; + + CrmResolvedTurbData(Int plev_, Int nelemd_, Int t1_, Int nelemd_todo_, Int np_todo_) : + PhysicsTestData({}, {}), plev(plev_), nelemd(nelemd_), t1(t1_), nelemd_todo(nelemd_todo_), np_todo(np_todo_) {} + + PTD_STD_DEF(CrmResolvedTurbData, 5, plev, nelemd, t1, nelemd_todo, np_todo); +}; + +struct IopDefaultOptsData { + // Inputs + Int plev; + + // Outputs + Real scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out; + std::string iopfile_out; + bool single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out; + + void randomize(std::mt19937_64& engine) {} + + IopDefaultOptsData() = default; +}; + +struct IopSetoptsData { + // Inputs + Int plev; + Real scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in; + std::string iopfile_in; + bool single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in; + + void randomize(std::mt19937_64& engine) {} + + IopSetoptsData() = default; +}; + +struct SetiopupdateInitData { + // Inputs + Int plev; + + void randomize(std::mt19937_64& engine) {} +}; + +struct SetiopupdateData { + // Inputs + Int plev; + + void randomize(std::mt19937_64& engine) {} +}; + +struct ReadiopdataData : public PhysicsTestData { + // Inputs + Int plev; + bool iop_update_phase1; + Real *hyam, *hybm; + + ReadiopdataData(Int plev_, bool iop_update_phase1_) : + PhysicsTestData({{ plev_ }}, {{ &hyam, &hybm }}), plev(plev_), iop_update_phase1(iop_update_phase1_) {} + + PTD_STD_DEF(ReadiopdataData, 2, plev, iop_update_phase1); +}; + +struct IopInthtData { + // Inputs + Int plev; + + void randomize(std::mt19937_64& engine) {} +}; + +// Glue functions to call fortran from from C++ with the Data struct + +void advance_iop_forcing(AdvanceIopForcingData& d); +void advance_iop_nudging(AdvanceIopNudgingData& d); +void advance_iop_subsidence(AdvanceIopSubsidenceData& d); +void iop_setinitial(IopSetinitialData& d); +void iop_broadcast(IopBroadcastData& d); +void apply_iop_forcing(ApplyIopForcingData& d); +void iop_domain_relaxation(IopDomainRelaxationData& d); +void crm_resolved_turb(CrmResolvedTurbData& d); +void iop_default_opts(IopDefaultOptsData& d); +void iop_setopts(IopSetoptsData& d); +void setiopupdate_init(SetiopupdateInitData& d); +void setiopupdate(SetiopupdateData& d); +void readiopdata(ReadiopdataData& d); +void iop_intht(IopInthtData& d); +extern "C" { // _f function decls + +void advance_iop_forcing_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, bool have_u, bool have_v, bool dp_crm, bool use_3dfrc, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* divt3d, Real* divq3d, Real* divt, Real* divq, Real* wfld, Real* uobs, Real* vobs, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* u_update, Real* v_update, Real* t_update, Real* q_update); + +void advance_iop_nudging_f(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* tobs, Real* qobs, + Real* hyai, Real* hyam, Real* hybi, Real* hybm, + Real* t_update, Real* q_update, Real* relaxt, Real* relaxq); + +void advance_iop_subsidence_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* wfld, Real* u_update, Real* v_update, Real* t_update, Real* q_update); + +void iop_setinitial_f(Int plev, Int pcnst, Int nelemd, Int np, Int nstep, Real psobs, bool use_replay, bool dynproc, bool have_t, bool have_q, bool have_ps, bool have_u, bool have_v, bool have_numliq, bool have_cldliq, bool have_numice, bool have_cldice, bool scm_zero_non_iop_tracers, bool is_first_restart_step, Real* qmin, Real* uobs, Real* vobs, Real* numliqobs, Real* numiceobs, Real* cldliqobs, Real* cldiceobs, Real* dx_short, tracer_t* tracers, element_t* elem, Real* dyn_dx_size, Real* tobs, Real* qobs); + +void iop_broadcast_f(); +void apply_iop_forcing_f(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t hybrid, timelevel_t tl, Int n, bool t_before_advance, Int nets, Int nete); +void iop_domain_relaxation_f(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt); +void crm_resolved_turb_f(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo); +void iop_default_opts_f(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out); +void iop_setopts_f(Real scmlat_in, Real scmlon_in, const char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in); +void setiopupdate_init_f(); +void setiopupdate_f(); +void readiopdata_f(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm); +void iop_intht_f(); +} // end _f function decls + +} // namespace dp +} // namespace scream + +#endif // SCREAM_DP_FUNCTIONS_F90_HPP diff --git a/components/eamxx/src/doubly-periodic/dp_iso_c.f90 b/components/eamxx/src/doubly-periodic/dp_iso_c.f90 new file mode 100644 index 000000000000..29d2fe55d4b2 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/dp_iso_c.f90 @@ -0,0 +1,156 @@ +module dp_iso_c + use iso_c_binding + implicit none + +#include "scream_config.f" +#ifdef SCREAM_DOUBLE_PRECISION +# define c_real c_double +#else +# define c_real c_float +#endif + +! +! This file contains bridges from scream c++ to DP fortran. +! + +contains + subroutine advance_iop_forcing_c(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) bind(C) + !use dp, only : advance_iop_forcing + + integer(kind=c_int) , value, intent(in) :: plev, pcnst + real(kind=c_real) , value, intent(in) :: scm_dt, ps_in + real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in, t_phys_frc + real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in + real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update + real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update + + !call advance_iop_forcing(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) + end subroutine advance_iop_forcing_c + subroutine advance_iop_nudging_c(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) bind(C) + !use dp, only : advance_iop_nudging + + integer(kind=c_int) , value, intent(in) :: plev + real(kind=c_real) , value, intent(in) :: scm_dt, ps_in + real(kind=c_real) , intent(in), dimension(plev) :: t_in, q_in + real(kind=c_real) , intent(out), dimension(plev) :: t_update, q_update, relaxt, relaxq + + !call advance_iop_nudging(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) + end subroutine advance_iop_nudging_c + subroutine advance_iop_subsidence_c(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) bind(C) + !use dp, only : advance_iop_subsidence + + integer(kind=c_int) , value, intent(in) :: plev, pcnst + real(kind=c_real) , value, intent(in) :: scm_dt, ps_in + real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in + real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in + real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update + real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update + + !call advance_iop_subsidence(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) + end subroutine advance_iop_subsidence_c + subroutine iop_setinitial_c(nelemd, elem) bind(C) + !use dp, only : iop_setinitial + + integer(kind=c_int) , value, intent(in) :: nelemd + type(c_ptr) , intent(inout), dimension(nelemd) :: elem + + !call iop_setinitial(nelemd, elem) + end subroutine iop_setinitial_c + subroutine iop_broadcast_c() bind(C) + !use dp, only : iop_broadcast + + !call iop_broadcast() + end subroutine iop_broadcast_c + subroutine apply_iop_forcing_c(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) bind(C) + !use dp, only : apply_iop_forcing + + integer(kind=c_int) , value, intent(in) :: nelemd, n, nets, nete + type(c_ptr) , intent(inout), dimension(nelemd) :: elem + type(c_ptr) , intent(inout) :: hvcoord + type(c_ptr) , intent(in) :: hybrid + type(c_ptr) , intent(in) :: tl + logical(kind=c_bool) , value, intent(in) :: t_before_advance + + !call apply_iop_forcing(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) + end subroutine apply_iop_forcing_c + subroutine iop_domain_relaxation_c(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) bind(C) + !use dp, only : iop_domain_relaxation + + integer(kind=c_int) , value, intent(in) :: nelemd, np, nlev, t1, nelemd_todo, np_todo + type(c_ptr) , intent(inout), dimension(nelemd) :: elem + type(c_ptr) , intent(in) :: hvcoord + type(c_ptr) , intent(in) :: hybrid + real(kind=c_real) , intent(inout), dimension(np, np, nlev) :: dp + real(kind=c_real) , value, intent(in) :: dt + + !call iop_domain_relaxation(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) + end subroutine iop_domain_relaxation_c + subroutine crm_resolved_turb_c(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) bind(C) + !use dp, only : crm_resolved_turb + + integer(kind=c_int) , value, intent(in) :: nelemd, t1, nelemd_todo, np_todo + type(c_ptr) , intent(inout), dimension(nelemd) :: elem + type(c_ptr) , intent(in) :: hvcoord + type(c_ptr) , intent(in) :: hybrid + + !call crm_resolved_turb(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) + end subroutine crm_resolved_turb_c + subroutine iop_default_opts_c(scmlat_out, scmlon_out, iopfile_out, & + single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, & + iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, & + iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, & + scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, & + scm_zero_non_iop_tracers_out) bind(C) + !use dp, only : iop_default_opts + + real(kind=c_real) , intent(out) :: scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out + type(c_ptr) , intent(out) :: iopfile_out + logical(kind=c_bool) , intent(out) :: single_column_out, & + scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, & + scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, & + dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out + + !call iop_default_opts(scmlat_out, scmlon_out, iopfile_out, single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, scm_zero_non_iop_tracers_out) + end subroutine iop_default_opts_c + subroutine iop_setopts_c(scmlat_in, scmlon_in, iopfile_in, & + single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, & + iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, & + iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, & + scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, & + scm_zero_non_iop_tracers_in) bind(C) + !use dp, only : iop_setopts + + real(kind=c_real) , value, intent(in) :: scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in + type(c_ptr) , intent(in) :: iopfile_in + logical(kind=c_bool) , value, intent(in) :: single_column_in, & + scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, & + scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, & + dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in + + !call iop_setopts(scmlat_in, scmlon_in, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, scm_zero_non_iop_tracers_in) + end subroutine iop_setopts_c + subroutine setiopupdate_init_c() bind(C) + !use dp, only : setiopupdate_init + + ! call setiopupdate_init() + end subroutine setiopupdate_init_c + subroutine setiopupdate_c() bind(C) + !use dp, only : setiopupdate + + !call setiopupdate() + end subroutine setiopupdate_c + subroutine readiopdata_c(plev, iop_update_phase1, hyam, hybm) bind(C) + !use dp, only : readiopdata + + integer(kind=c_int) , value, intent(in) :: plev + logical(kind=c_bool) , value, intent(in) :: iop_update_phase1 + real(kind=c_real) , intent(in), dimension(plev) :: hyam, hybm + + !call readiopdata(plev, iop_update_phase1, hyam, hybm) + end subroutine readiopdata_c + subroutine iop_intht_c() bind(C) + !use dp, only : iop_intht + + !call iop_intht() + end subroutine iop_intht_c +end module dp_iso_c diff --git a/components/eamxx/src/doubly-periodic/dp_iso_f.f90 b/components/eamxx/src/doubly-periodic/dp_iso_f.f90 new file mode 100644 index 000000000000..98c48961aa4a --- /dev/null +++ b/components/eamxx/src/doubly-periodic/dp_iso_f.f90 @@ -0,0 +1,123 @@ +module dp_iso_f + use iso_c_binding + implicit none + +#include "scream_config.f" +#ifdef SCREAM_DOUBLE_PRECISION +# define c_real c_double +#else +# define c_real c_float +#endif + +! +! This file contains bridges from DP fortran to scream c++. +! + +interface + + subroutine advance_iop_forcing_f(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: plev, pcnst + real(kind=c_real) , value, intent(in) :: scm_dt, ps_in + real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in, t_phys_frc + real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in + real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update + real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update + end subroutine advance_iop_forcing_f + subroutine advance_iop_nudging_f(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: plev + real(kind=c_real) , value, intent(in) :: scm_dt, ps_in + real(kind=c_real) , intent(in), dimension(plev) :: t_in, q_in + real(kind=c_real) , intent(out), dimension(plev) :: t_update, q_update, relaxt, relaxq + end subroutine advance_iop_nudging_f + subroutine advance_iop_subsidence_f(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: plev, pcnst + real(kind=c_real) , value, intent(in) :: scm_dt, ps_in + real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in + real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in + real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update + real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update + end subroutine advance_iop_subsidence_f + subroutine iop_setinitial_f(nelemd, elem) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: nelemd + type(c_ptr) , intent(inout), dimension(nelemd) :: elem + end subroutine iop_setinitial_f + subroutine iop_broadcast_f() bind(C) + use iso_c_binding + + + end subroutine iop_broadcast_f + subroutine apply_iop_forcing_f(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: nelemd, n, nets, nete + type(c_ptr) , intent(inout), dimension(nelemd) :: elem + type(c_ptr) , intent(inout) :: hvcoord + type(c_ptr) , intent(in) :: hybrid + type(c_ptr) , intent(in) :: tl + logical(kind=c_bool) , value, intent(in) :: t_before_advance + end subroutine apply_iop_forcing_f + subroutine iop_domain_relaxation_f(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: nelemd, np, nlev, t1, nelemd_todo, np_todo + type(c_ptr) , intent(inout), dimension(nelemd) :: elem + type(c_ptr) , intent(in) :: hvcoord + type(c_ptr) , intent(in) :: hybrid + real(kind=c_real) , intent(inout), dimension(np, np, nlev) :: dp + real(kind=c_real) , value, intent(in) :: dt + end subroutine iop_domain_relaxation_f + subroutine crm_resolved_turb_f(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: nelemd, t1, nelemd_todo, np_todo + type(c_ptr) , intent(inout), dimension(nelemd) :: elem + type(c_ptr) , intent(in) :: hvcoord + type(c_ptr) , intent(in) :: hybrid + end subroutine crm_resolved_turb_f + subroutine iop_default_opts_f(scmlat_out, scmlon_out, iopfile_out, single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, scm_zero_non_iop_tracers_out) bind(C) + use iso_c_binding + + real(kind=c_real) , intent(out) :: scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out + type(c_ptr) , intent(out) :: iopfile_out + logical(kind=c_bool) , intent(out) :: single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out + end subroutine iop_default_opts_f + subroutine iop_setopts_f(scmlat_in, scmlon_in, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, scm_zero_non_iop_tracers_in) bind(C) + use iso_c_binding + + real(kind=c_real) , value, intent(in) :: scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in + type(c_ptr) , intent(in) :: iopfile_in + logical(kind=c_bool) , value, intent(in) :: single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in + end subroutine iop_setopts_f + subroutine setiopupdate_init_f() bind(C) + use iso_c_binding + + + end subroutine setiopupdate_init_f + subroutine setiopupdate_f() bind(C) + use iso_c_binding + + + end subroutine setiopupdate_f + subroutine readiopdata_f(plev, iop_update_phase1, hyam, hybm) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: plev + logical(kind=c_bool) , value, intent(in) :: iop_update_phase1 + real(kind=c_real) , intent(in), dimension(plev) :: hyam, hybm + end subroutine readiopdata_f + subroutine iop_intht_f() bind(C) + use iso_c_binding + + + end subroutine iop_intht_f +end interface + +end module dp_iso_f diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp new file mode 100644 index 000000000000..a0f560796a4d --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_advance_iop_forcing_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing advance_iop_forcing on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp new file mode 100644 index 000000000000..6e5fab7e6090 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_advance_iop_nudging_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing advance_iop_nudging on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp new file mode 100644 index 000000000000..1fe6c79a160a --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_advance_iop_subsidence_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing advance_iop_subsidence on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp b/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp new file mode 100644 index 000000000000..cb138eb9b048 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_apply_iop_forcing_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing apply_iop_forcing on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp b/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp new file mode 100644 index 000000000000..6921dc81cc5a --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_crm_resolved_turb_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing crm_resolved_turb on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp new file mode 100644 index 000000000000..083d19b97afd --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_iop_broadcast_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing iop_broadcast on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp new file mode 100644 index 000000000000..30fae390febb --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_iop_default_opts_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing iop_default_opts on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp new file mode 100644 index 000000000000..2b8aa9a09add --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_iop_domain_relaxation_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing iop_domain_relaxation on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp new file mode 100644 index 000000000000..b68587342a5c --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_iop_intht_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing iop_intht on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp new file mode 100644 index 000000000000..8641cfa0704a --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_iop_setfield_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing iop_setfield on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp new file mode 100644 index 000000000000..9195f4f6cf3b --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_iop_setinitial_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing iop_setinitial on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp new file mode 100644 index 000000000000..e46aff90bdc6 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_iop_setopts_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing iop_setopts on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp b/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp new file mode 100644 index 000000000000..4289e61b9848 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_readiopdata_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing readiopdata on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp b/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp new file mode 100644 index 000000000000..56e7084e4760 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_setiopupdate_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing setiopupdate on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp b/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp new file mode 100644 index 000000000000..45f26f184665 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp @@ -0,0 +1,14 @@ +#include "impl/dp_setiopupdate_init_impl.hpp" + +namespace scream { +namespace dp { + +/* + * Explicit instantiation for doing setiopupdate_init on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace dp +} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp new file mode 100644 index 000000000000..5220d14e290b --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp @@ -0,0 +1,158 @@ +#ifndef DP_ADVANCE_IOP_FORCING_IMPL_HPP +#define DP_ADVANCE_IOP_FORCING_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp advance_iop_forcing. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::plevs0( + // Input arguments + const Int& nver, + const Scalar& ps, + const uview_1d& hyai, + const uview_1d& hyam, + const uview_1d& hybi, + const uview_1d& hybm, + // Kokkos stuff + const MemberType& team, + // Output arguments + const uview_1d& pint, + const uview_1d& pmid, + const uview_1d& pdel) +{ + const auto ps0 = C::P0; + const Int nver_pack = ekat::npack(nver); + + // Set interface pressures + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nver_pack), [&] (Int k) { + pint(k) = hyai(k)*ps0 + hybi(k)*ps; + pmid(k) = hyam(k)*ps0 + hybm(k)*ps; + }); + + // Set midpoint pressures and layer thicknesses + const auto pint_s = scalarize(pint); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nver_pack), [&] (Int k) { + Spack spint, spint_1; + IntSmallPack range_pack1 = ekat::range(k*Spack::n); + auto range_pack2_p1_safe = range_pack1; + range_pack2_p1_safe.set(range_pack1 > nver-1, nver-1); + ekat::index_and_shift<1>(pint_s, range_pack2_p1_safe, spint, spint_1); + pdel(k) = spint_1 - spint; + }); +} + +template +KOKKOS_FUNCTION +void Functions::advance_iop_forcing( + // Input arguments + const Int& plev, + const Int& pcnst, + const bool& have_u, + const bool& have_v, + const bool& dp_crm, + const bool& use_3dfrc, + const Scalar& scm_dt, + const Scalar& ps_in, + const uview_1d& u_in, + const uview_1d& v_in, + const uview_1d& t_in, + const uview_2d& q_in, + const uview_1d& t_phys_frc, + const uview_1d& divt3d, + const uview_2d& divq3d, + const uview_1d& divt, + const uview_2d& divq, + const uview_1d& wfld, + const uview_1d& uobs, + const uview_1d& vobs, + const uview_1d& hyai, + const uview_1d& hyam, + const uview_1d& hybi, + const uview_1d& hybm, + // Kokkos stuff + const MemberType& team, + const Workspace& workspace, + // Output arguments + const uview_1d& u_update, + const uview_1d& v_update, + const uview_1d& t_update, + const uview_2d& q_update) +{ + // Local variables + uview_1d + pmidm1, // pressure at model levels + pintm1, // pressure at model interfaces (dim=plev+1) + pdelm1; // pdel(k) = pint (k+1)-pint (k) + workspace.template take_many_contiguous_unsafe<3>( + {"pmidm1", "pintm1", "pdelm1"}, + {&pmidm1, &pintm1, &pdelm1}); + + // Get vertical level profiles + plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); + + //////////////////////////////////////////////////////////// + // Advance T and Q due to large scale forcing + + uview_1d t_lsf; // storage for temperature large scale forcing + uview_2d q_lsf; // storage for moisture large scale forcing + + if (use_3dfrc) { + t_lsf = divt3d; + q_lsf = divq3d; + } + else { + t_lsf = divt; + q_lsf = divq; + } + + const Int plev_pack = ekat::npack(plev); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { + // Initialize thermal expansion term to zero. This term is only + // considered if using the preq-x dycore and if three dimensional + // forcing is not provided by IOP forcing file. + Spack t_expan = 0; + t_update(k) = t_in(k) + t_expan + scm_dt*(t_phys_frc(k) + t_lsf(k)); + for (Int m = 0; m < pcnst; ++m) { + q_update(m, k) = q_in(m, k) + scm_dt*q_lsf(m, k); + } + }); + + //////////////////////////////////////////////////////////// + // Set U and V fields + + uview_1d u_src, v_src; + + if (have_v && have_u && !dp_crm) { + u_src = uobs; + v_src = vobs; + } + else { + u_src = u_in; + v_src = v_in; + } + + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { + u_update(k) = u_src(k); + v_update(k) = v_src(k); + }); + + workspace.template release_many_contiguous<3>( + {&pmidm1, &pintm1, &pdelm1}); +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp new file mode 100644 index 000000000000..cf0f160209a3 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp @@ -0,0 +1,76 @@ +#ifndef DP_ADVANCE_IOP_NUDGING_IMPL_HPP +#define DP_ADVANCE_IOP_NUDGING_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp advance_iop_nudging. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::advance_iop_nudging( + // Input arguments + const Int& plev, + const Scalar& scm_dt, + const Scalar& ps_in, + const uview_1d& t_in, + const uview_1d& q_in, + const uview_1d& tobs, + const uview_1d& qobs, + const uview_1d& hyai, + const uview_1d& hyam, + const uview_1d& hybi, + const uview_1d& hybm, + // Kokkos stuff + const MemberType& team, + const Workspace& workspace, + // Output arguments + const uview_1d& t_update, + const uview_1d& q_update, + const uview_1d& relaxt, + const uview_1d& relaxq) +{ + // Local variables + uview_1d + pmidm1, // pressure at model levels + pintm1, // pressure at model interfaces (dim=plev+1) + pdelm1, // pdel(k) = pint (k+1)-pint (k) + rtau; + workspace.template take_many_contiguous_unsafe<4>( + {"pmidm1", "pintm1", "pdelm1", "rtau"}, + {&pmidm1, &pintm1, &pdelm1, &rtau}); + + // Get vertical level profiles + plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); + + const Int plev_pack = ekat::npack(plev); + constexpr Scalar iop_nudge_tq_low = DPC::iop_nudge_tq_low; + constexpr Scalar iop_nudge_tq_high = DPC::iop_nudge_tq_high; + constexpr Scalar iop_nudge_tscale = DPC::iop_nudge_tscale; + + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { + relaxt(k) = 0; + relaxq(k) = 0; + + const auto condition = pmidm1(k) <= iop_nudge_tq_low*100 + && + pmidm1(k) >= iop_nudge_tq_high*100; + rtau(k).set(condition, ekat::impl::max(scm_dt, iop_nudge_tscale)); + relaxt(k).set(condition, (t_in(k) - tobs(k))/rtau(k)); + relaxq(k).set(condition, (q_in(k) - qobs(k))/rtau(k)); + + t_update(k).set(condition, t_in(k) + relaxt(k)*scm_dt); + q_update(k).set(condition, q_in(k) + relaxq(k)*scm_dt); + }); +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp new file mode 100644 index 000000000000..3fc7a53320a8 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp @@ -0,0 +1,164 @@ +#ifndef DP_ADVANCE_IOP_SUBSIDENCE_IMPL_HPP +#define DP_ADVANCE_IOP_SUBSIDENCE_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace dp { + +/* + * Implementation of dp advance_iop_subsidence. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::do_advance_iop_subsidence_update( + const Int& k, + const Int& plev, + const Spack& fac, + const Spack& swfldint, + const Spack& swfldint_p1, + const uview_1d& in, + const uview_1d& in_s, + const uview_1d& update) +{ + Spack sin, sin_p1, sin_m1; + + auto range_pack1 = ekat::range(k*Spack::n); + auto range_pack2_m1_safe = range_pack1; + auto range_pack2_p1_safe = range_pack1; + range_pack2_m1_safe.set(range_pack1 < 1, 1); // don't want the shift to go below zero. we mask out that result anyway + range_pack2_p1_safe.set(range_pack1 > plev-2, plev-2); // don't want the shift to go beyond pack. + ekat::index_and_shift<-1>(in_s, range_pack2_m1_safe, sin, sin_m1); + ekat::index_and_shift< 1>(in_s, range_pack2_p1_safe, sin, sin_p1); + + update(k) = in(k) - fac*(swfldint_p1*(sin_p1 - sin) + swfldint*(sin - sin_m1)); +} + +template +KOKKOS_FUNCTION +void Functions::advance_iop_subsidence( + const Int& plev, + const Int& pcnst, + const Scalar& scm_dt, + const Scalar& ps_in, + const uview_1d& u_in, + const uview_1d& v_in, + const uview_1d& t_in, + const uview_2d& q_in, + const uview_1d& hyai, + const uview_1d& hyam, + const uview_1d& hybi, + const uview_1d& hybm, + const uview_1d& wfld, + const MemberType& team, + const Workspace& workspace, + const uview_1d& u_update, + const uview_1d& v_update, + const uview_1d& t_update, + const uview_2d& q_update) +{ + // Local variables + uview_1d + pmidm1, // pressure at model levels + pintm1, // pressure at model interfaces (dim=plev+1) + pdelm1, // pdel(k) = pint (k+1)-pint (k) + wfldint;// (dim=plev+1) + workspace.template take_many_contiguous_unsafe<4>( + {"pmidm1", "pintm1", "pdelm1", "wfldint"}, + {&pmidm1, &pintm1, &pdelm1, &wfldint}); + + const Int plev_pack = ekat::npack(plev); + + // Get vertical level profiles + plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); + + // Scalarize a bunch of views that need shift operations + auto pmidm1_s = scalarize(pmidm1); + auto pintm1_s = scalarize(pintm1); + auto wfld_s = scalarize(wfld); + auto wfldint_s = scalarize(wfldint); + auto u_in_s = scalarize(u_in); + auto v_in_s = scalarize(v_in); + auto t_in_s = scalarize(t_in); + + wfldint_s(0) = 0; + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { + Spack spmidm1, spmidm1_m1, spintm1, spintm1_m1, swfld, swfld_m1; + auto range_pack1 = ekat::range(k*Spack::n); + auto range_pack2 = range_pack1; + range_pack2.set(range_pack1 < 1, 1); // don't want the shift to go below zero. we mask out that result anyway + ekat::index_and_shift<-1>(pmidm1_s, range_pack2, spmidm1, spmidm1_m1); + ekat::index_and_shift<-1>(pintm1_s, range_pack2, spintm1, spintm1_m1); + ekat::index_and_shift<-1>(wfld_s, range_pack2, swfld, swfld_m1); + Spack weight = (spintm1 - spintm1_m1) / (spmidm1 - spmidm1_m1); + wfldint(k) = (1 - weight)*swfld_m1 + weight*swfld; + }); + wfldint_s(plev) = 0; + + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { + Spack swfldint, swfldint_p1; + auto range_pack1 = ekat::range(k*Spack::n); + auto range_pack2 = range_pack1; + range_pack2.set(range_pack1 > plev-1, plev-1); + + ekat::index_and_shift<1>(wfldint_s, range_pack2, swfldint, swfldint_p1); + + Spack fac = scm_dt/(2 * pdelm1(k)); + + do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, u_in, u_in_s, u_update); + do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, v_in, v_in_s, v_update); + do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, t_in, t_in_s, t_update); + + for (Int m = 0; m < pcnst; ++m) { + // Grab m-th subview of q stuff + auto q_update_sub = ekat::subview(q_update, m); + auto q_in_sub = ekat::subview(q_in, m); + auto q_in_sub_s = scalarize(q_in_sub); + do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, q_in_sub, q_in_sub_s, q_update_sub); + } + }); + + // Top and bottom levels next + Kokkos::Array bot_top = {0, plev-1}; + for (Int i = 0; i < 2; ++i) { + const auto k = bot_top[i]; + const auto pack_idx = ekat::npack(k+1) - 1; + const auto s_idx = k % Spack::n; + const auto idx1 = k == 0 ? k+1 : k; + + Scalar fac = scm_dt/(2 * pdelm1(pack_idx)[s_idx]); + u_update(pack_idx)[s_idx] = u_in_s(k) - fac*(wfldint_s(idx1)*(u_in_s(idx1) - u_in_s(idx1-1))); + v_update(pack_idx)[s_idx] = v_in_s(k) - fac*(wfldint_s(idx1)*(v_in_s(idx1) - v_in_s(idx1-1))); + t_update(pack_idx)[s_idx] = t_in_s(k) - fac*(wfldint_s(idx1)*(t_in_s(idx1) - t_in_s(idx1-1))); + + for (Int m = 0; m < pcnst; ++m) { + auto q_update_sub = ekat::subview(q_update, m); + auto q_in_sub = ekat::subview(q_in, m); + auto q_in_sub_s = scalarize(q_in_sub); + + q_update_sub(pack_idx)[s_idx] = q_in_sub_s(k) - fac*(wfldint_s(idx1)*(q_in_sub_s(idx1) - q_in_sub_s(idx1-1))); + } + } + + // thermal expansion term due to LS vertical advection + constexpr Scalar rair = C::Rair; + constexpr Scalar cpair = C::Cpair; + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { + t_update(k) = t_update(k) + scm_dt*wfld(k)*t_in(k)*rair/(cpair*pmidm1(k)); + }); + + workspace.template release_many_contiguous<4>( + {&pmidm1, &pintm1, &pdelm1, &wfldint}); +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp new file mode 100644 index 000000000000..cacae8b46ddd --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_APPLY_IOP_FORCING_IMPL_HPP +#define DP_APPLY_IOP_FORCING_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp apply_iop_forcing. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::apply_iop_forcing(const Int& nelemd, const uview_1d& elem, hvcoord_t& hvcoord, const hybrid_t& hybrid, const timelevel_t& tl, const Int& n, const bool& t_before_advance, const Int& nets, const Int& nete) +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp new file mode 100644 index 000000000000..2e17f7a43c5b --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_CRM_RESOLVED_TURB_IMPL_HPP +#define DP_CRM_RESOLVED_TURB_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp crm_resolved_turb. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::crm_resolved_turb(const Int& nelemd, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const Int& nelemd_todo, const Int& np_todo) +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp new file mode 100644 index 000000000000..ead8ec78a2b6 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_IOP_BROADCAST_IMPL_HPP +#define DP_IOP_BROADCAST_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp iop_broadcast. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::iop_broadcast() +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp new file mode 100644 index 000000000000..e5687d7558bb --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp @@ -0,0 +1,24 @@ +#ifndef DP_IOP_DEFAULT_OPTS_IMPL_HPP +#define DP_IOP_DEFAULT_OPTS_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp iop_default_opts. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +void Functions::iop_default_opts(Spack& scmlat_out, Spack& scmlon_out, std::string& iopfile_out, bool& single_column_out, bool& scm_iop_srf_prop_out, bool& iop_nudge_tq_out, bool& iop_nudge_uv_out, Spack& iop_nudge_tq_low_out, Spack& iop_nudge_tq_high_out, Spack& iop_nudge_tscale_out, bool& scm_observed_aero_out, bool& iop_dosubsidence_out, bool& scm_multcols_out, bool& dp_crm_out, Spack& iop_perturb_high_out, bool& precip_off_out, bool& scm_zero_non_iop_tracers_out) +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp new file mode 100644 index 000000000000..ad36a9a0ce51 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_IOP_DOMAIN_RELAXATION_IMPL_HPP +#define DP_IOP_DOMAIN_RELAXATION_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp iop_domain_relaxation. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::iop_domain_relaxation(const Int& nelemd, const Int& np, const Int& nlev, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const uview_1d& dp, const Int& nelemd_todo, const Int& np_todo, const Spack& dt) +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp new file mode 100644 index 000000000000..e03631f94cb7 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_IOP_INTHT_IMPL_HPP +#define DP_IOP_INTHT_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp iop_intht. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::iop_intht() +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp new file mode 100644 index 000000000000..ee8991c472ba --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_IOP_SETFIELD_IMPL_HPP +#define DP_IOP_SETFIELD_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp iop_setfield. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::iop_setfield(const Int& nelemd, const uview_1d& elem, const bool& iop_update_phase1) +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp new file mode 100644 index 000000000000..854ced7dd48d --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp @@ -0,0 +1,207 @@ +#ifndef DP_IOP_SETINITIAL_IMPL_HPP +#define DP_IOP_SETINITIAL_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +#include "Context.hpp" +#include "TimeLevel.hpp" + +namespace scream { +namespace dp { + +/* + * Implementation of dp iop_setinitial. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +void Functions::iop_setinitial( + const Int& plev, + const Int& pcnst, + const Int& nelemd, + const Int& np, + const Int& nstep, + const bool& use_replay, + const bool& dynproc, + const bool& have_t, + const bool& have_q, + const bool& have_ps, + const bool& have_u, + const bool& have_v, + const bool& have_numliq, + const bool& have_cldliq, + const bool& have_numice, + const bool& have_cldice, + const bool& scm_zero_non_iop_tracers, + const bool& is_first_restart_step, + const uview_1d& qmin, + const uview_1d& uobs, + const uview_1d& vobs, + const uview_1d& numliqobs, + const uview_1d& numiceobs, + const uview_1d& cldliqobs, + const uview_1d& cldiceobs, + const Scalar& psobs, + const uview_1d& dx_short, + Scalar& dyn_dx_size, + tracer_t& tracers, + element_t& elem, + const uview_1d& tobs, + const uview_1d& qobs) +{ + // Made these up + + //FieldGroup g = ...; + //const auto& names = g.m_info->m_field_names; + constexpr Int inumliq = 1; + constexpr Int inumice = 2; + constexpr Int icldliq = 3; + constexpr Int icldice = 4; + + const Int plev_packs = ekat::npack(plev); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_packs); + + assert(np <= NP); + //assert(plev_packs <= Homme::NUM_LEV); + assert(tracers.inited()); + assert(elem.inited()); + + // Get time info + const Int n0 = Homme::Context::singleton().get().n0; + + if (!use_replay && nstep == 0 && dynproc) { + + Kokkos::parallel_for( + "iop_setinitial loop", + policy, + KOKKOS_LAMBDA(const MemberType& team) { + + // We cannot support parallelism at the element level because the thelev + // computation of tobs is dependent on prior iterations that may have alterted tobs + for (Int ie = 0; ie < nelemd; ++ie) { + for (Int j = 0; j < np; ++j) { + for (Int i = 0; i < np; ++i) { + + // Find level where tobs is no longer zero + Int thelev=-1; + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(team, plev_packs), [&] (int plev_pack, Int& pmin) { + auto zmask = tobs(plev_pack) != 0; + if (zmask.any()) { + pmin = plev_pack; + } + }, Kokkos::Min(thelev)); + + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, thelev+1), [&] (int k) { + auto zmask = k < thelev ? Smask(true) : tobs(k) == 0; + vector_simd for (Int p = 0; p < Spack::n; ++p) { + if (zmask[p]) { + tobs(k)[p] = elem.m_forcing.m_ft(ie,i,j,k)[p]; + qobs(k)[p] = tracers.Q(ie,0,i,j,k)[p]; // Tracer index 0 is qobs + } + } + }); + + if (scm_zero_non_iop_tracers) { + for (Int cix = 0; cix < pcnst; ++cix) { + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, plev_packs), [&] (int k) { + tracers.Q(ie, cix, i, j, k) = qmin(cix); + }); + } + } + + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, thelev, plev_packs), [&] (int k) { + auto zmask = k > thelev ? Smask(true) : tobs(k) != 0; + if (have_t) { + vector_simd for (Int p = 0; p < Spack::n; ++p) { + if (zmask[p]) { + elem.m_forcing.m_ft(ie,i,j,k)[p] = tobs(k)[p]; + } + } + } + if (have_q) { + vector_simd for (Int p = 0; p < Spack::n; ++p) { + if (zmask[p]) { + tracers.Q(ie,0,i,j,k)[p] = qobs(k)[p]; + } + } + } + }); + + if (have_ps) { + elem.m_state.m_ps_v(ie, n0, i, j) = psobs; + } + + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, plev_packs), [&] (int k) { + if (have_u) { + vector_simd for (Int p = 0; p < Spack::n; ++p) { + elem.m_state.m_v(ie, 0, 0, i, j, k)[p] = uobs(k)[p]; // [2] is 0 for u + } + } + if (have_v) { + vector_simd for (Int p = 0; p < Spack::n; ++p) { + elem.m_state.m_v(ie, 0, 1, i, j, k)[p] = vobs(k)[p]; // [2] is 1 for v + } + } + if (have_numliq) { + vector_simd for (Int p = 0; p < Spack::n; ++p) { + tracers.Q(ie, inumliq, i, j, k)[p] = numliqobs(k)[p]; + } + } + if (have_cldliq) { + vector_simd for (Int p = 0; p < Spack::n; ++p) { + tracers.Q(ie, icldliq, i, j, k)[p] = cldliqobs(k)[p]; + } + } + if (have_numice) { + vector_simd for (Int p = 0; p < Spack::n; ++p) { + tracers.Q(ie, inumice, i, j, k)[p] = numiceobs(k)[p]; + } + } + if (have_cldice) { + vector_simd for (Int p = 0; p < Spack::n; ++p) { + tracers.Q(ie, icldice, i, j, k)[p] = cldiceobs(k)[p]; + } + } + + // If DP-CRM mode we do NOT want to write over the dy-core vertical + // velocity with the large-scale one. wfld is used in forecast.F90 + // for the compuation of the large-scale subsidence. + elem.m_derived.m_omega_p(ie, i, j, k) = 0; + }); + } + } + } + }); + } + + // If DP-CRM mode then SHOC/CLUBB needs to know about grid + // length size. The calculations of this based on a sphere in the + // SHOC and CLUBB interefaces are not valid for a planar grid, thus + // save the grid length from the dycore. Note that planar dycore + // only supports uniform grids, thus we only save one value. + // Set this if it is the first time step or the first restart step + if ( (nstep == 0 || is_first_restart_step) && dynproc) { + // for (Int ie = 0; ie < nelemd; ++ie) { + // dyn_dx_size = dx_short(ie) * 1000; // why not just grab the last ie? + // } + Kokkos::parallel_reduce( + "iop_setinitial loop", + policy, + KOKKOS_LAMBDA(const MemberType& team, Scalar& dyn) { + const Int ie = team.league_rank(); + if (ie == nelemd-1) { + dyn = dx_short(nelemd-1) * 1000; + } + }, dyn_dx_size); + } +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp new file mode 100644 index 000000000000..5483abdd6a41 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp @@ -0,0 +1,24 @@ +#ifndef DP_IOP_SETOPTS_IMPL_HPP +#define DP_IOP_SETOPTS_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp iop_setopts. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +void Functions::iop_setopts(const Spack& scmlat_in, const Spack& scmlon_in, const std::string& iopfile_in, const bool& single_column_in, const bool& scm_iop_srf_prop_in, const bool& iop_nudge_tq_in, const bool& iop_nudge_uv_in, const Spack& iop_nudge_tq_low_in, const Spack& iop_nudge_tq_high_in, const Spack& iop_nudge_tscale_in, const bool& scm_observed_aero_in, const bool& iop_dosubsidence_in, const bool& scm_multcols_in, const bool& dp_crm_in, const Spack& iop_perturb_high_in, const bool& precip_off_in, const bool& scm_zero_non_iop_tracers_in) +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp new file mode 100644 index 000000000000..3cd83bfcdb06 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_READIOPDATA_IMPL_HPP +#define DP_READIOPDATA_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp readiopdata. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::readiopdata(const Int& plev, const bool& iop_update_phase1, const uview_1d& hyam, const uview_1d& hybm) +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp new file mode 100644 index 000000000000..78e1bfe7340c --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_SETIOPUPDATE_IMPL_HPP +#define DP_SETIOPUPDATE_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp setiopupdate. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::setiopupdate() +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp new file mode 100644 index 000000000000..bc9ba8d41c61 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp @@ -0,0 +1,25 @@ +#ifndef DP_SETIOPUPDATE_INIT_IMPL_HPP +#define DP_SETIOPUPDATE_INIT_IMPL_HPP + +#include "dp_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace dp { + +/* + * Implementation of dp setiopupdate_init. Clients should NOT + * #include this file, but include dp_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::setiopupdate_init() +{ + // TODO + // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed +} + +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt b/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt new file mode 100644 index 000000000000..32e2883a22f6 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt @@ -0,0 +1,27 @@ +INCLUDE (ScreamUtils) + +set(DP_TESTS_SRCS + dp_unit_tests.cpp + dp_advance_iop_forcing_tests.cpp + dp_advance_iop_nudging_tests.cpp + dp_advance_iop_subsidence_tests.cpp + dp_iop_setinitial_tests.cpp + dp_iop_broadcast_tests.cpp + dp_apply_iop_forcing_tests.cpp + dp_iop_domain_relaxation_tests.cpp + dp_crm_resolved_turb_tests.cpp + dp_iop_default_opts_tests.cpp + dp_iop_setopts_tests.cpp + dp_setiopupdate_init_tests.cpp + dp_setiopupdate_tests.cpp + dp_readiopdata_tests.cpp + dp_iop_intht_tests.cpp + ) # DP_TESTS_SRCS + +# NOTE: tests inside this if statement won't be built in a baselines-only build +if (NOT SCREAM_BASELINES_ONLY) + CreateUnitTest(dp_tests "${DP_TESTS_SRCS}" + LIBS dp + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + ) +endif() diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp new file mode 100644 index 000000000000..d1f5ea89a42e --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp @@ -0,0 +1,116 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestAdvanceIopForcing { + + static void run_bfb() + { + auto engine = setup_random_test(); + + AdvanceIopForcingData f90_data[] = { + // plev, pcnst, scm_dt, ps_in, have_u, have_v, dp_crm, use_3dfrc + AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, true, true), + AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, true, false), + AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, false, true), + + AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, true, true), + AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, true, false), + AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, false, true), + + AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, true, true), + AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, true, false), + AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, false, true), + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopForcingData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + AdvanceIopForcingData cxx_data[num_runs] = { + AdvanceIopForcingData(f90_data[0]), + AdvanceIopForcingData(f90_data[1]), + AdvanceIopForcingData(f90_data[2]), + AdvanceIopForcingData(f90_data[3]), + AdvanceIopForcingData(f90_data[4]), + AdvanceIopForcingData(f90_data[5]), + AdvanceIopForcingData(f90_data[6]), + AdvanceIopForcingData(f90_data[7]), + AdvanceIopForcingData(f90_data[8]), + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + advance_iop_forcing(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + d.template transpose(); // _f expects data in fortran layout + advance_iop_forcing_f(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.have_u, d.have_v, d.dp_crm, d.use_3dfrc, d.u_in, d.v_in, d.t_in, d.q_in, d.t_phys_frc, d.divt3d, d.divq3d, d.divt, d.divq, d.wfld, d.uobs, d.vobs, d.hyai, d.hyam, d.hybi, d.hybm, d.u_update, d.v_update, d.t_update, d.q_update); + d.template transpose(); // go back to C layout + } + + // We can't call into fortran. Due to all the dependencies it has, it's not possible + // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. +#if 0 + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + AdvanceIopForcingData& d_f90 = f90_data[i]; + AdvanceIopForcingData& d_cxx = cxx_data[i]; + for (Int k = 0; k < d_f90.total(d_f90.u_update); ++k) { + REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.u_update)); + REQUIRE(d_f90.u_update[k] == d_cxx.u_update[k]); + REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.v_update)); + REQUIRE(d_f90.v_update[k] == d_cxx.v_update[k]); + REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.t_update)); + REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); + } + for (Int k = 0; k < d_f90.total(d_f90.q_update); ++k) { + REQUIRE(d_f90.total(d_f90.q_update) == d_cxx.total(d_cxx.q_update)); + REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); + } + + } + } +#endif + + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("advance_iop_forcing_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopForcing; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp new file mode 100644 index 000000000000..02f9362eb7a3 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp @@ -0,0 +1,98 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestAdvanceIopNudging { + + static void run_bfb() + { + auto engine = setup_random_test(); + + AdvanceIopNudgingData f90_data[] = { + // plev, scm_dt, ps_in + AdvanceIopNudgingData(72, 0.1, 1000), + AdvanceIopNudgingData(27, 0.1, 1000), + AdvanceIopNudgingData(32, 0.1, 1000), + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopNudgingData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + AdvanceIopNudgingData cxx_data[] = { + AdvanceIopNudgingData(f90_data[0]), + AdvanceIopNudgingData(f90_data[1]), + AdvanceIopNudgingData(f90_data[2]), + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + advance_iop_nudging(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + advance_iop_nudging_f(d.plev, d.scm_dt, d.ps_in, d.t_in, d.q_in, d.tobs, d.qobs, + d.hyai, d.hyam, d.hybi, d.hybm, + d.t_update, d.q_update, d.relaxt, d.relaxq); + } + + // We can't call into fortran. Due to all the dependencies it has, it's not possible + // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. +#if 0 + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + AdvanceIopNudgingData& d_f90 = f90_data[i]; + AdvanceIopNudgingData& d_cxx = cxx_data[i]; + for (Int k = 0; k < d_f90.total(d_f90.t_update); ++k) { + REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.t_update)); + REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); + REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.q_update)); + REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); + REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.relaxt)); + REQUIRE(d_f90.relaxt[k] == d_cxx.relaxt[k]); + REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.relaxq)); + REQUIRE(d_f90.relaxq[k] == d_cxx.relaxq[k]); + } + + } + } +#endif + } // run_bfb +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("advance_iop_nudging_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopNudging; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp new file mode 100644 index 000000000000..a430135f75be --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp @@ -0,0 +1,107 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestAdvanceIopSubsidence { + + static void run_bfb() + { + auto engine = setup_random_test(); + + AdvanceIopSubsidenceData f90_data[] = { + // plev, pcnst, scm_dt, ps_in + AdvanceIopSubsidenceData(72, 10, 0.1, 1000.0), + AdvanceIopSubsidenceData(72, 7, 0.1, 1000.0), + AdvanceIopSubsidenceData(27, 10, 0.1, 1000.0), + AdvanceIopSubsidenceData(27, 7, 0.1, 1000.0), + AdvanceIopSubsidenceData(32, 10, 0.1, 1000.0), + AdvanceIopSubsidenceData(32, 7, 0.1, 1000.0), + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopSubsidenceData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + AdvanceIopSubsidenceData cxx_data[num_runs] = { + AdvanceIopSubsidenceData(f90_data[0]), + AdvanceIopSubsidenceData(f90_data[1]), + AdvanceIopSubsidenceData(f90_data[2]), + AdvanceIopSubsidenceData(f90_data[3]), + AdvanceIopSubsidenceData(f90_data[4]), + AdvanceIopSubsidenceData(f90_data[5]), + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + advance_iop_subsidence(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + d.template transpose(); // _f expects data in fortran layout + advance_iop_subsidence_f(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.hyai, d.hyam, d.hybi, d.hybm, d.wfld, d.u_update, d.v_update, d.t_update, d.q_update); + d.template transpose(); // go back to C layout + } + + // We can't call into fortran. Due to all the dependencies it has, it's not possible + // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. +#if 0 + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + AdvanceIopSubsidenceData& d_f90 = f90_data[i]; + AdvanceIopSubsidenceData& d_cxx = cxx_data[i]; + for (Int k = 0; k < d_f90.total(d_f90.u_update); ++k) { + REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.u_update)); + REQUIRE(d_f90.u_update[k] == d_cxx.u_update[k]); + REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.v_update)); + REQUIRE(d_f90.v_update[k] == d_cxx.v_update[k]); + REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.t_update)); + REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); + } + for (Int k = 0; k < d_f90.total(d_f90.q_update); ++k) { + REQUIRE(d_f90.total(d_f90.q_update) == d_cxx.total(d_cxx.q_update)); + REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); + } + + } + } +#endif + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("advance_iop_subsidence_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopSubsidence; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp new file mode 100644 index 000000000000..3e85e30974bc --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp @@ -0,0 +1,78 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestApplyIopForcing { + + static void run_bfb() + { + auto engine = setup_random_test(); + + ApplyIopForcingData f90_data[] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(ApplyIopForcingData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + ApplyIopForcingData cxx_data[] = { + // TODO + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + apply_iop_forcing(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + apply_iop_forcing_f(d.nelemd, d.elem, &d.hvcoord, d.hybrid, d.tl, d.n, d.t_before_advance, d.nets, d.nete); + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + ApplyIopForcingData& d_f90 = f90_data[i]; + ApplyIopForcingData& d_cxx = cxx_data[i]; + //REQUIRE(d_f90.hvcoord == d_cxx.hvcoord); + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("apply_iop_forcing_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestApplyIopForcing; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp new file mode 100644 index 000000000000..f56d24880606 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp @@ -0,0 +1,78 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestCrmResolvedTurb { + + static void run_bfb() + { + auto engine = setup_random_test(); + + CrmResolvedTurbData f90_data[] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(CrmResolvedTurbData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + CrmResolvedTurbData cxx_data[] = { + // TODO + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + crm_resolved_turb(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + crm_resolved_turb_f(d.nelemd, d.elem, d.hvcoord, d.hybrid, d.t1, d.nelemd_todo, d.np_todo); + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + CrmResolvedTurbData& d_f90 = f90_data[i]; + CrmResolvedTurbData& d_cxx = cxx_data[i]; + + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("crm_resolved_turb_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestCrmResolvedTurb; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp new file mode 100644 index 000000000000..0ffb8f63e5a5 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp @@ -0,0 +1,85 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestIopBroadcast { + + static void run_bfb() + { + auto engine = setup_random_test(); + + IopBroadcastData f90_data[max_pack_size]; //= { + // // TODO + // }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopBroadcastData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + +#if 0 + // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // inout data is in original state + view_1d cxx_device("cxx_device", max_pack_size); + const auto cxx_host = Kokkos::create_mirror_view(cxx_device); + std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + Kokkos::deep_copy(cxx_device, cxx_host); + + // Get data from fortran + for (auto& d : f90_data) { + iop_broadcast(d); + } + + // Get data from cxx. Run iop_broadcast from a kernel and copy results back to host + // Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { + // const Int offset = i * Spack::n; + + + + + // Functions::iop_broadcast(); + + + // }); + + Kokkos::deep_copy(cxx_host, cxx_device); + + // Verify BFB results + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + IopBroadcastData& d_f90 = f90_data[i]; + IopBroadcastData& d_cxx = cxx_host[i]; + } + } +#endif + } // run_bfb +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("iop_broadcast_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopBroadcast; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp new file mode 100644 index 000000000000..622ca26a7264 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp @@ -0,0 +1,106 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestIopDefaultOpts { + + static void run_bfb() + { + auto engine = setup_random_test(); + + IopDefaultOptsData f90_data[max_pack_size] = { + // TODO + }; + + static constexpr Int num_runs = 0; //sizeof(f90_data) / sizeof(IopDefaultOptsData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // inout data is in original state + using host_view = typename view_1d::host_mirror_type; + host_view cxx_host("cxx_host", max_pack_size); + std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + + // Get data from fortran + for (auto& d : f90_data) { + iop_default_opts(d); + } + + // Get data from cxx. Run iop_default_opts from a kernel and copy results back to host + for (Int i = 0; i < num_test_itrs; ++i) { + const Int offset = i * Spack::n; + + // Init outputs + Spack iop_nudge_tq_high_out(0), iop_nudge_tq_low_out(0), iop_nudge_tscale_out(0), iop_perturb_high_out(0), scmlat_out(0), scmlon_out(0); + + Functions::iop_default_opts(scmlat_out, scmlon_out, cxx_host(0).iopfile_out, cxx_host(0).single_column_out, cxx_host(0).scm_iop_srf_prop_out, cxx_host(0).iop_nudge_tq_out, cxx_host(0).iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, cxx_host(0).scm_observed_aero_out, cxx_host(0).iop_dosubsidence_out, cxx_host(0).scm_multcols_out, cxx_host(0).dp_crm_out, iop_perturb_high_out, cxx_host(0).precip_off_out, cxx_host(0).scm_zero_non_iop_tracers_out); + + // Copy spacks back into cxx_host view + for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { + cxx_host(vs).iop_nudge_tq_high_out = iop_nudge_tq_high_out[s]; + cxx_host(vs).iop_nudge_tq_low_out = iop_nudge_tq_low_out[s]; + cxx_host(vs).iop_nudge_tscale_out = iop_nudge_tscale_out[s]; + cxx_host(vs).iop_perturb_high_out = iop_perturb_high_out[s]; + cxx_host(vs).scmlat_out = scmlat_out[s]; + cxx_host(vs).scmlon_out = scmlon_out[s]; + } + } + + // Verify BFB results + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + IopDefaultOptsData& d_f90 = f90_data[i]; + IopDefaultOptsData& d_cxx = cxx_host[i]; + REQUIRE(d_f90.scmlat_out == d_cxx.scmlat_out); + REQUIRE(d_f90.scmlon_out == d_cxx.scmlon_out); + REQUIRE(d_f90.iopfile_out == d_cxx.iopfile_out); + REQUIRE(d_f90.single_column_out == d_cxx.single_column_out); + REQUIRE(d_f90.scm_iop_srf_prop_out == d_cxx.scm_iop_srf_prop_out); + REQUIRE(d_f90.iop_nudge_tq_out == d_cxx.iop_nudge_tq_out); + REQUIRE(d_f90.iop_nudge_uv_out == d_cxx.iop_nudge_uv_out); + REQUIRE(d_f90.iop_nudge_tq_low_out == d_cxx.iop_nudge_tq_low_out); + REQUIRE(d_f90.iop_nudge_tq_high_out == d_cxx.iop_nudge_tq_high_out); + REQUIRE(d_f90.iop_nudge_tscale_out == d_cxx.iop_nudge_tscale_out); + REQUIRE(d_f90.scm_observed_aero_out == d_cxx.scm_observed_aero_out); + REQUIRE(d_f90.iop_dosubsidence_out == d_cxx.iop_dosubsidence_out); + REQUIRE(d_f90.scm_multcols_out == d_cxx.scm_multcols_out); + REQUIRE(d_f90.dp_crm_out == d_cxx.dp_crm_out); + REQUIRE(d_f90.iop_perturb_high_out == d_cxx.iop_perturb_high_out); + REQUIRE(d_f90.precip_off_out == d_cxx.precip_off_out); + REQUIRE(d_f90.scm_zero_non_iop_tracers_out == d_cxx.scm_zero_non_iop_tracers_out); + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("iop_default_opts_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopDefaultOpts; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp new file mode 100644 index 000000000000..ed570c6767a4 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp @@ -0,0 +1,84 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestIopDomainRelaxation { + + static void run_bfb() + { + auto engine = setup_random_test(); + + IopDomainRelaxationData f90_data[] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopDomainRelaxationData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + IopDomainRelaxationData cxx_data[] = { + // TODO + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + iop_domain_relaxation(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + d.transpose(); // _f expects data in fortran layout + iop_domain_relaxation_f(d.nelemd, d.np, d.nlev, d.elem, d.hvcoord, d.hybrid, d.t1, d.dp, d.nelemd_todo, d.np_todo, d.dt); + d.transpose(); // go back to C layout + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + IopDomainRelaxationData& d_f90 = f90_data[i]; + IopDomainRelaxationData& d_cxx = cxx_data[i]; + for (Int k = 0; k < d_f90.total(d_f90.dp); ++k) { + REQUIRE(d_f90.total(d_f90.dp) == d_cxx.total(d_cxx.dp)); + REQUIRE(d_f90.dp[k] == d_cxx.dp[k]); + } + + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("iop_domain_relaxation_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopDomainRelaxation; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp new file mode 100644 index 000000000000..f214ebb49961 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp @@ -0,0 +1,84 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestIopIntht { + + static void run_bfb() + { + auto engine = setup_random_test(); + + IopInthtData f90_data[max_pack_size] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopInthtData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // inout data is in original state + view_1d cxx_device("cxx_device", max_pack_size); + const auto cxx_host = Kokkos::create_mirror_view(cxx_device); + std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + Kokkos::deep_copy(cxx_device, cxx_host); + + // Get data from fortran + for (auto& d : f90_data) { + iop_intht(d); + } + + // Get data from cxx. Run iop_intht from a kernel and copy results back to host + Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { + const Int offset = i * Spack::n; + + + + + Functions::iop_intht(); + + + }); + + Kokkos::deep_copy(cxx_host, cxx_device); + + // Verify BFB results + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + IopInthtData& d_f90 = f90_data[i]; + IopInthtData& d_cxx = cxx_host[i]; + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("iop_intht_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopIntht; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp new file mode 100644 index 000000000000..04921cbcbfb8 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp @@ -0,0 +1,78 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestIopSetfield { + + static void run_bfb() + { + auto engine = setup_random_test(); + + IopSetfieldData f90_data[] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetfieldData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + IopSetfieldData cxx_data[] = { + // TODO + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + iop_setfield(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + iop_setfield_f(d.nelemd, d.elem, d.iop_update_phase1); + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + IopSetfieldData& d_f90 = f90_data[i]; + IopSetfieldData& d_cxx = cxx_data[i]; + + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("iop_setfield_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetfield; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp new file mode 100644 index 000000000000..d7e1d6d7a7df --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp @@ -0,0 +1,102 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestIopSetinitial { + + static void run_bfb() + { + auto engine = setup_random_test(); + + IopSetinitialData f90_data[] = { + // plev, pcnst, nelemd, np, nstep, psobs, use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step + IopSetinitialData(72 , 5 , 10 , 2 , 10, 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), + IopSetinitialData(72 , 5 , 10 , 2 , 10, 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), + IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), + IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), + IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , false), + IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , false), + IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), + IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), + IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, true , true , true , true , true , true , true , true , true , true , true , false , true), + IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, false , true , true , true , true , true , true , true , true , true , true , false , true), + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetinitialData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + IopSetinitialData cxx_data[] = { + IopSetinitialData(f90_data[0]), + IopSetinitialData(f90_data[1]), + IopSetinitialData(f90_data[2]), + IopSetinitialData(f90_data[3]), + IopSetinitialData(f90_data[4]), + IopSetinitialData(f90_data[5]), + IopSetinitialData(f90_data[6]), + IopSetinitialData(f90_data[7]), + IopSetinitialData(f90_data[8]), + IopSetinitialData(f90_data[9]), + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + iop_setinitial(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + d.init(); + iop_setinitial_f(d.plev, d.pcnst, d.nelemd, d.np, d.nstep, d.psobs, d.use_replay, d.dynproc, d.have_t, d.have_q, d.have_ps, d.have_u, d.have_v, d.have_numliq, d.have_cldliq, d.have_numice, d.have_cldice, d.scm_zero_non_iop_tracers, d.is_first_restart_step, d.qmin, d.uobs, d.vobs, d.numliqobs, d.numiceobs, d.cldliqobs, d.cldiceobs, d.dx_short, &d.tracers, &d.elem, &d.dyn_dx_size, d.tobs, d.qobs); + } + + // We can't call into fortran. Due to all the dependencies it has, it's not possible + // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. +#if 0 + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + IopSetinitialData& d_f90 = f90_data[i]; + IopSetinitialData& d_cxx = cxx_data[i]; + + } + } +#endif + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("iop_setinitial_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetinitial; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp new file mode 100644 index 000000000000..d24f54fb634a --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp @@ -0,0 +1,87 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestIopSetopts { + + static void run_bfb() + { + auto engine = setup_random_test(); + + IopSetoptsData f90_data[max_pack_size] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetoptsData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // inout data is in original state + using host_view = typename view_1d::host_mirror_type; + host_view cxx_host("cxx_host", max_pack_size); + std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + + // Get data from fortran + for (auto& d : f90_data) { + iop_setopts(d); + } + + // Get data from cxx. Run iop_setopts from a kernel and copy results back to host + for (Int i = 0; i < num_test_itrs; ++i) { + const Int offset = i * Spack::n; + + // Init pack inputs + Spack iop_nudge_tq_high_in, iop_nudge_tq_low_in, iop_nudge_tscale_in, iop_perturb_high_in, scmlat_in, scmlon_in; + for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { + iop_nudge_tq_high_in[s] = cxx_host(vs).iop_nudge_tq_high_in; + iop_nudge_tq_low_in[s] = cxx_host(vs).iop_nudge_tq_low_in; + iop_nudge_tscale_in[s] = cxx_host(vs).iop_nudge_tscale_in; + iop_perturb_high_in[s] = cxx_host(vs).iop_perturb_high_in; + scmlat_in[s] = cxx_host(vs).scmlat_in; + scmlon_in[s] = cxx_host(vs).scmlon_in; + } + + Functions::iop_setopts(scmlat_in, scmlon_in, cxx_host(0).iopfile_in, cxx_host(0).single_column_in, cxx_host(0).scm_iop_srf_prop_in, cxx_host(0).iop_nudge_tq_in, cxx_host(0).iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, cxx_host(0).scm_observed_aero_in, cxx_host(0).iop_dosubsidence_in, cxx_host(0).scm_multcols_in, cxx_host(0).dp_crm_in, iop_perturb_high_in, cxx_host(0).precip_off_in, cxx_host(0).scm_zero_non_iop_tracers_in); + } + + // Verify BFB results + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + IopSetoptsData& d_f90 = f90_data[i]; + IopSetoptsData& d_cxx = cxx_host[i]; + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("iop_setopts_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetopts; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp new file mode 100644 index 000000000000..bef219811845 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp @@ -0,0 +1,78 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestReadiopdata { + + static void run_bfb() + { + auto engine = setup_random_test(); + + ReadiopdataData f90_data[] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(ReadiopdataData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + ReadiopdataData cxx_data[] = { + // TODO + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + readiopdata(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + readiopdata_f(d.plev, d.iop_update_phase1, d.hyam, d.hybm); + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + ReadiopdataData& d_f90 = f90_data[i]; + ReadiopdataData& d_cxx = cxx_data[i]; + + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("readiopdata_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestReadiopdata; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp new file mode 100644 index 000000000000..b60fc5f8d4cf --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp @@ -0,0 +1,84 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestSetiopupdateInit { + + static void run_bfb() + { + auto engine = setup_random_test(); + + SetiopupdateInitData f90_data[max_pack_size] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(SetiopupdateInitData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // inout data is in original state + view_1d cxx_device("cxx_device", max_pack_size); + const auto cxx_host = Kokkos::create_mirror_view(cxx_device); + std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + Kokkos::deep_copy(cxx_device, cxx_host); + + // Get data from fortran + for (auto& d : f90_data) { + setiopupdate_init(d); + } + + // Get data from cxx. Run setiopupdate_init from a kernel and copy results back to host + Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { + const Int offset = i * Spack::n; + + + + + Functions::setiopupdate_init(); + + + }); + + Kokkos::deep_copy(cxx_host, cxx_device); + + // Verify BFB results + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + SetiopupdateInitData& d_f90 = f90_data[i]; + SetiopupdateInitData& d_cxx = cxx_host[i]; + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("setiopupdate_init_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestSetiopupdateInit; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp new file mode 100644 index 000000000000..800aaab005bf --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp @@ -0,0 +1,84 @@ +#include "catch2/catch.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "doubly-periodic/dp_functions.hpp" +#include "doubly-periodic/dp_functions_f90.hpp" + +#include "dp_unit_tests_common.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestSetiopupdate { + + static void run_bfb() + { + auto engine = setup_random_test(); + + SetiopupdateData f90_data[max_pack_size] = { + // TODO + }; + + static constexpr Int num_runs = sizeof(f90_data) / sizeof(SetiopupdateData); + + // Generate random input data + // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // inout data is in original state + view_1d cxx_device("cxx_device", max_pack_size); + const auto cxx_host = Kokkos::create_mirror_view(cxx_device); + std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + Kokkos::deep_copy(cxx_device, cxx_host); + + // Get data from fortran + for (auto& d : f90_data) { + setiopupdate(d); + } + + // Get data from cxx. Run setiopupdate from a kernel and copy results back to host + Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { + const Int offset = i * Spack::n; + + + + + Functions::setiopupdate(); + + + }); + + Kokkos::deep_copy(cxx_host, cxx_device); + + // Verify BFB results + if (SCREAM_BFB_TESTING) { + for (Int i = 0; i < num_runs; ++i) { + SetiopupdateData& d_f90 = f90_data[i]; + SetiopupdateData& d_cxx = cxx_host[i]; + } + } + } // run_bfb + +}; + +} // namespace unit_test +} // namespace dp +} // namespace scream + +namespace { + +TEST_CASE("setiopupdate_bfb", "[dp]") +{ + using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestSetiopupdate; + + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp new file mode 100644 index 000000000000..bfd865a1d924 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp @@ -0,0 +1,26 @@ +#include "catch2/catch.hpp" + +#include "dp_unit_tests_common.hpp" + +#include "dp_functions.hpp" +#include "dp_functions_f90.hpp" + +#include "share/scream_types.hpp" + +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "ekat/util/ekat_arch.hpp" + +#include +#include +#include +#include + +namespace scream { +namespace dp { +namespace unit_test { + +}//namespace unit_test +}//namespace dp +}//namespace scream + diff --git a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp b/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp new file mode 100644 index 000000000000..b2a85e9e5db7 --- /dev/null +++ b/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp @@ -0,0 +1,79 @@ +#ifndef DP_UNIT_TESTS_COMMON_HPP +#define DP_UNIT_TESTS_COMMON_HPP + +#include "dp_functions.hpp" +#include "share/scream_types.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "share/util/scream_setup_random_test.hpp" + +namespace scream { +namespace dp { +namespace unit_test { + +/* + * Unit test infrastructure for dp unit tests. + * + * dp entities can friend scream::dp::unit_test::UnitWrap to give unit tests + * access to private members. + * + * All unit test impls should be within an inner struct of UnitWrap::UnitTest for + * easy access to useful types. + */ + +struct UnitWrap { + + template + struct UnitTest : public KokkosTypes { + + using Device = D; + using MemberType = typename KokkosTypes::MemberType; + using TeamPolicy = typename KokkosTypes::TeamPolicy; + using RangePolicy = typename KokkosTypes::RangePolicy; + using ExeSpace = typename KokkosTypes::ExeSpace; + + template + using view_1d = typename KokkosTypes::template view_1d; + template + using view_2d = typename KokkosTypes::template view_2d; + template + using view_3d = typename KokkosTypes::template view_3d; + + template + using uview_1d = typename ekat::template Unmanaged >; + + using Functions = scream::dp::Functions; + using Scalar = typename Functions::Scalar; + using Spack = typename Functions::Spack; + using Pack = typename Functions::Pack; + using IntSmallPack = typename Functions::IntSmallPack; + using Smask = typename Functions::Smask; + using C = typename Functions::C; + + static constexpr Int max_pack_size = 16; + static constexpr Int num_test_itrs = max_pack_size / Spack::n; + + // Put struct decls here + struct TestAdvanceIopForcing; + struct TestAdvanceIopNudging; + struct TestAdvanceIopSubsidence; + struct TestIopSetinitial; + struct TestIopBroadcast; + struct TestApplyIopForcing; + struct TestIopDomainRelaxation; + struct TestCrmResolvedTurb; + struct TestIopDefaultOpts; + struct TestIopSetopts; + struct TestSetiopupdateInit; + struct TestSetiopupdate; + struct TestReadiopdata; + struct TestIopIntht; + }; + +}; + + +} // namespace unit_test +} // namespace dp +} // namespace scream + +#endif diff --git a/components/eamxx/src/dynamics/CMakeLists.txt b/components/eamxx/src/dynamics/CMakeLists.txt index 706ecf0c1efd..ea675dd58dee 100644 --- a/components/eamxx/src/dynamics/CMakeLists.txt +++ b/components/eamxx/src/dynamics/CMakeLists.txt @@ -1,6 +1,5 @@ if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") add_subdirectory(homme) - set (SCREAM_HAS_HOMME TRUE CACHE INTERNAL "Signal that Homme is compiled") elseif("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "NONE") message ("SCREAM_DYNAMICS_DYCORE set to 'NONE'. Scream won't enable any test/code that needs dynamics.\n") else() diff --git a/components/eamxx/src/dynamics/homme/CMakeLists.txt b/components/eamxx/src/dynamics/homme/CMakeLists.txt index 26725d13b13e..3cf34d3eb88a 100644 --- a/components/eamxx/src/dynamics/homme/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/CMakeLists.txt @@ -114,8 +114,25 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) target_compile_definitions(${hommeLibName} PUBLIC HOMMEXX_CONFIG_IS_CMAKE SCREAM) # Link to cime's csm_share lib target_link_libraries (${hommeLibName} csm_share) - target_link_libraries (${hommeLibName} composec++) - SetCudaFlags(${hommeLibName}) + if (HOMME_ENABLE_COMPOSE) + target_link_libraries (${hommeLibName} composec++) + endif() + + string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci) + if ((SCREAM_MACHINE STREQUAL "ascent" OR SCREAM_MACHINE STREQUAL "pm-gpu") AND CMAKE_BUILD_TYPE_ci STREQUAL "debug") + # --fmad=false is causing nondeterminism in RRTMGP on Ascent and pm-gpu, + # perhaps due to an nvcc bug. Provide a FLAGS entry to prevent + # SetCudaFlags from adding --fmad=false. Use -UNDEBUG as an (inert) entry + # for FLAGS so that cmake_parse_arguments defines SCF_FLAGS. + SetCudaFlags(${hommeLibName} CUDA_LANG FLAGS -UNDEBUG) + else() + # In the non-debug case, I want to make sure anything else that is done in + # SetCudaFlags continues to be done. Right now, I don't think any of it + # matters, but given the generality of the name "SetCudaFlags", it's + # possible something that matters will be added in the future. + SetCudaFlags(${hommeLibName} CUDA_LANG) + endif() + SetOmpFlags(${hommeLibName}) ##################################### @@ -126,9 +143,9 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) set (SCREAM_DYNAMICS_SRC_DIR ${SCREAM_SRC_DIR}/dynamics/homme) set (SCREAM_DYNAMICS_SOURCES - ${SCREAM_DYNAMICS_SRC_DIR}/atmosphere_dynamics.cpp - ${SCREAM_DYNAMICS_SRC_DIR}/atmosphere_dynamics_fv_phys.cpp - ${SCREAM_DYNAMICS_SRC_DIR}/atmosphere_dynamics_rayleigh_friction.cpp + ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_process_interface.cpp + ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_fv_phys.cpp + ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_rayleigh_friction.cpp ${SCREAM_DYNAMICS_SRC_DIR}/physics_dynamics_remapper.cpp ${SCREAM_DYNAMICS_SRC_DIR}/homme_grids_manager.cpp ${SCREAM_DYNAMICS_SRC_DIR}/interface/homme_context_mod.F90 @@ -142,7 +159,8 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) # Create library set (dynLibName scream_${hommeLibName}) add_library(${dynLibName} ${SCREAM_DYNAMICS_SOURCES}) - target_link_libraries(${dynLibName} scream_share scream_io ${hommeLibName}) + target_compile_definitions(${dynLibName} PUBLIC EAMXX_HAS_HOMME) + target_link_libraries(${dynLibName} PUBLIC scream_share scream_io ${hommeLibName}) get_target_property(modulesDir ${hommeLibName} Fortran_MODULE_DIRECTORY) set_target_properties(${dynLibName} PROPERTIES Fortran_MODULE_DIRECTORY ${modulesDir}) target_include_directories(${dynLibName} PUBLIC ${modulesDir}) diff --git a/components/eamxx/src/dynamics/homme/atmosphere_dynamics_fv_phys.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp similarity index 89% rename from components/eamxx/src/dynamics/homme/atmosphere_dynamics_fv_phys.cpp rename to components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp index 55c9b884f75e..4e815c7266a9 100644 --- a/components/eamxx/src/dynamics/homme/atmosphere_dynamics_fv_phys.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp @@ -1,4 +1,4 @@ -#include "atmosphere_dynamics.hpp" +#include "eamxx_homme_process_interface.hpp" // HOMMEXX Includes #include "Context.hpp" @@ -10,6 +10,7 @@ // Scream includes #include "share/field/field_manager.hpp" +#include "share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp" #include "dynamics/homme/homme_dimensions.hpp" // Ekat includes @@ -89,7 +90,7 @@ static void copy_prev (const int ncols, const int npacks, FM(icol,1,ilev) = uv(icol,1,ilev); }); }); - Kokkos::fence(); + Kokkos::fence(); } void HommeDynamics::fv_phys_dyn_to_fv_phys (const bool restart) { @@ -106,7 +107,7 @@ void HommeDynamics::fv_phys_dyn_to_fv_phys (const bool restart) { constexpr int NGP = HOMMEXX_NP; const int nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const auto npg = m_phys_grid_pgN*m_phys_grid_pgN; - GllFvRemapTmp t; + GllFvRemapTmp t; t.T_mid = Homme::ExecView("T_mid_tmp", nelem, npg, npacks*N); t.horiz_winds = Homme::ExecView("horiz_winds_tmp", nelem, npg, 2, npacks*N); // Really need just the first tracer. @@ -171,7 +172,7 @@ void HommeDynamics::remap_dyn_to_fv_phys (GllFvRemapTmp* t) const { const auto nq = get_group_out("tracers").m_bundle->get_view().extent_int(1); assert(get_field_out("T_mid", gn).get_view().extent_int(0) == nelem*npg); assert(get_field_out("horiz_winds", gn).get_view().extent_int(1) == 2); - + const auto ps = Homme::GllFvRemap::Phys1T( get_field_out("ps", gn).get_view().data(), nelem, npg); @@ -223,40 +224,18 @@ void HommeDynamics::remap_fv_phys_to_dyn () const { const auto q = Homme::GllFvRemap::CPhys3T( get_group_in("tracers", gn).m_bundle->get_view().data(), nelem, npg, nq, nlev); - + gfr.run_fv_phys_to_dyn(time_idx, T, uv, q); Kokkos::fence(); gfr.run_fv_phys_to_dyn_dss(); Kokkos::fence(); } -// TODO [rrtmgp active gases] This is to address issue #1782. It supports option -// 1 in that issue. These fv_phys_rrtmgp_active_gases_* routines can be removed -// once rrtmgp active_gases initialization is treated properly. - -struct TraceGasesWorkaround { - bool restart{false}; - std::shared_ptr remapper; - std::vector active_gases; // other than h2o -}; - -static TraceGasesWorkaround s_tgw; - -void fv_phys_rrtmgp_active_gases_init (const ekat::ParameterList& p) { - const auto& v = p.sublist("atmosphere_processes").sublist("physics") - .sublist("rrtmgp").get>("active_gases"); - if (ekat::contains(v, "o3")) { - s_tgw.active_gases.push_back("o3_volume_mix_ratio"); - } -} - -void fv_phys_rrtmgp_active_gases_set_restart (const bool restart) { - s_tgw.restart = restart; -} - +// See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp void HommeDynamics ::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& gm) { - if (s_tgw.restart) return; // always false b/c it hasn't been set yet + auto& trace_gases_workaround = TraceGasesWorkaround::singleton(); + if (trace_gases_workaround.is_restart()) return; // always false b/c it hasn't been set yet using namespace ekat::units; using namespace ShortFieldTagsNames; auto molmol = mol/mol; @@ -267,23 +246,25 @@ ::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& g const auto pnc = m_phys_grid->get_num_local_dofs(); const auto nlev = m_cgll_grid->get_num_vertical_levels(); constexpr int ps = SCREAM_SMALL_PACK_SIZE; - for (const auto& e : s_tgw.active_gases) { + for (const auto& e : trace_gases_workaround.get_active_gases()) { add_field(e, FieldLayout({COL,LEV},{rnc,nlev}), molmol, rgn, ps); // 'Updated' rather than just 'Computed' so that it gets written to the // restart file. add_field(e, FieldLayout({COL,LEV},{pnc,nlev}), molmol, pgn, ps); } - s_tgw.remapper = gm->create_remapper(m_cgll_grid, m_dyn_grid); + trace_gases_workaround.set_remapper(gm->create_remapper(m_cgll_grid, m_dyn_grid)); } +// See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp void HommeDynamics::fv_phys_rrtmgp_active_gases_remap () { // Note re: restart: Ideally, we'd know if we're restarting before having to // call add_field above. However, we only find out after. Because the pg2 // field was declared Updated, it will read the restart data. But we don't // actually want to remap from CGLL to pg2 now. So if restarting, just do the // cleanup part at the end. + auto& trace_gases_workaround = TraceGasesWorkaround::singleton(); const auto& rgn = m_cgll_grid->name(); - if (not s_tgw.restart) { + if (not trace_gases_workaround.is_restart()) { using namespace ShortFieldTagsNames; const auto& dgn = m_dyn_grid ->name(); const auto& pgn = m_phys_grid->name(); @@ -293,23 +274,23 @@ void HommeDynamics::fv_phys_rrtmgp_active_gases_remap () { const int nelem = m_dyn_grid->get_num_local_dofs()/ngll; { // CGLL -> DGLL const auto nlev = m_dyn_grid->get_num_vertical_levels(); - for (const auto& e : s_tgw.active_gases) + for (const auto& e : trace_gases_workaround.get_active_gases()) create_helper_field(e, {EL,GP,GP,LEV}, {nelem,NGP,NGP,nlev}, dgn); - auto& r = s_tgw.remapper; + auto r = trace_gases_workaround.get_remapper(); r->registration_begins(); - for (const auto& e : s_tgw.active_gases) + for (const auto& e : trace_gases_workaround.get_active_gases()) r->register_field(get_field_in(e, rgn), m_helper_fields.at(e)); r->registration_ends(); r->remap(true); - s_tgw.remapper = nullptr; + trace_gases_workaround.erase_remapper(); } { // DGLL -> PGN const auto& c = Homme::Context::singleton(); auto& gfr = c.get(); const auto time_idx = c.get().n0; - for (const auto& e : s_tgw.active_gases) { + for (const auto& e : trace_gases_workaround.get_active_gases()) { const auto& f_dgll = m_helper_fields.at(e); - const auto& f_phys = get_field_out(e, pgn); + auto& f_phys = get_field_out(e, pgn); const auto& v_dgll = f_dgll.get_view(); const auto& v_phys = f_phys.get_view(); assert(v_dgll.extent_int(0) == nelem and @@ -321,14 +302,16 @@ void HommeDynamics::fv_phys_rrtmgp_active_gases_remap () { v_phys.data(), nelem, npg, 1, v_phys.extent_int(1)); gfr.remap_tracer_dyn_to_fv_phys(time_idx, 1, in_dgll, out_phys); Kokkos::fence(); + + f_phys.get_header().get_tracking().update_time_stamp(timestamp()); } } } // Done with all of these, so remove them. - s_tgw.remapper = nullptr; - for (const auto& e : s_tgw.active_gases) + trace_gases_workaround.erase_remapper(); + for (const auto& e : trace_gases_workaround.get_active_gases()) m_helper_fields.erase(e); - for (const auto& e : s_tgw.active_gases) + for (const auto& e : trace_gases_workaround.get_active_gases()) remove_field(e, rgn); } diff --git a/components/eamxx/src/dynamics/homme/atmosphere_dynamics.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp similarity index 98% rename from components/eamxx/src/dynamics/homme/atmosphere_dynamics.cpp rename to components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index d307ec78dd6c..c535829fbfab 100644 --- a/components/eamxx/src/dynamics/homme/atmosphere_dynamics.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -1,4 +1,4 @@ -#include "atmosphere_dynamics.hpp" +#include "eamxx_homme_process_interface.hpp" // HOMMEXX Includes #include "Context.hpp" @@ -234,6 +234,11 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m add_internal_field (m_helper_fields.at("w_int_dyn").subfield(1,tl.n0,true)); } + // The output manager pulls from the atm process fields. Add + // helper fields for the case that a user request output. + add_internal_field (m_helper_fields.at("omega_dyn")); + add_internal_field (m_helper_fields.at("phis_dyn")); + if (not fv_phys_active()) { // Dynamics backs out tendencies from the states, and passes those to Homme. // After Homme completes, we remap the updated state to the ref grid. Thus, @@ -281,8 +286,10 @@ size_t HommeDynamics::requested_buffer_size_in_bytes() const auto& esf = c.create_if_not_there(num_elems); fbm.request_size(esf.requested_buffer_size()); } else { +#ifdef HOMME_ENABLE_COMPOSE auto& ct = c.create_if_not_there(num_elems); fbm.request_size(ct.requested_buffer_size()); +#endif } if (need_dirk) { // Create dirk functor only if needed @@ -419,6 +426,12 @@ void HommeDynamics::initialize_impl (const RunType run_type) restart_homme_state (); } + // Since we just inited them, ensure p_mid/p_int (dry and wet) timestamps are valid + get_field_out("p_int") .get_header().get_tracking().update_time_stamp(timestamp()); + get_field_out("p_mid") .get_header().get_tracking().update_time_stamp(timestamp()); + get_field_out("p_dry_int").get_header().get_tracking().update_time_stamp(timestamp()); + get_field_out("p_dry_mid").get_header().get_tracking().update_time_stamp(timestamp()); + // Complete homme model initialization prim_init_model_f90 (); @@ -433,7 +446,7 @@ void HommeDynamics::initialize_impl (const RunType run_type) // dynamics helper fields Computed and output on the dynamics grid. Worse // for I/O but better for device memory. const auto& rgn = m_cgll_grid->name(); - for (const auto& f : {"horiz_winds", "T_mid", "ps", "phis"}) + for (const auto& f : {"horiz_winds", "T_mid", "ps", "phis", "pseudo_density"}) remove_field(f, rgn); remove_group("tracers", rgn); fv_phys_rrtmgp_active_gases_remap(); @@ -1209,6 +1222,13 @@ void HommeDynamics::initialize_homme_state () { update_pressure (m_phys_grid); } + // If "Instant" averaging type is used for output, + // an initial output is performed before AD processes + // are run. If omega_dyn output is requested, it will + // not have valid computed values for this initial + // output. Set to zero avoid potential FPE. + get_internal_field("omega_dyn").deep_copy(0); + // Copy IC states on all timelevel slices copy_dyn_states_to_all_timelevels (); diff --git a/components/eamxx/src/dynamics/homme/atmosphere_dynamics.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp similarity index 98% rename from components/eamxx/src/dynamics/homme/atmosphere_dynamics.hpp rename to components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index 346951b8bf8c..6f6da5321484 100644 --- a/components/eamxx/src/dynamics/homme/atmosphere_dynamics.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -81,7 +81,7 @@ class HommeDynamics : public AtmosphereProcess void fv_phys_dyn_to_fv_phys(const bool restart = false); void fv_phys_pre_process(); void fv_phys_post_process(); - // See [rrtmgp active gases] in atmosphere_dynamics_fv_phys.cpp. + // See [rrtmgp active gases] in eamxx_homme_fv_phys.cpp. void fv_phys_rrtmgp_active_gases_init(const std::shared_ptr& gm); void fv_phys_rrtmgp_active_gases_remap(); @@ -95,7 +95,7 @@ class HommeDynamics : public AtmosphereProcess struct GllFvRemapTmp; void remap_dyn_to_fv_phys(GllFvRemapTmp* t = nullptr) const; void remap_fv_phys_to_dyn() const; - + protected: void run_impl (const double dt); void finalize_impl (); diff --git a/components/eamxx/src/dynamics/homme/atmosphere_dynamics_rayleigh_friction.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp similarity index 96% rename from components/eamxx/src/dynamics/homme/atmosphere_dynamics_rayleigh_friction.cpp rename to components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp index 17391567b9ae..120dce865db6 100644 --- a/components/eamxx/src/dynamics/homme/atmosphere_dynamics_rayleigh_friction.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp @@ -1,4 +1,4 @@ -#include "atmosphere_dynamics.hpp" +#include "eamxx_homme_process_interface.hpp" // Scream includes #include "share/util/scream_common_physics_functions.hpp" @@ -18,7 +18,7 @@ void HommeDynamics::rayleigh_friction_init() // If m_raytau0==0, then no Rayleigh friction is applied. Return. if (m_raytau0 == 0) return; - // Input file is read in 1-based indexing, convert + // Input file is read in 1-based indexing, convert // to 0-based for computations m_rayk0 -= 1; @@ -45,7 +45,7 @@ void HommeDynamics::rayleigh_friction_init() // locals for lambda captures to avoid issues on GPU auto otau = m_otau; auto rayk0 = m_rayk0; - + Kokkos::parallel_for(KT::RangePolicy(0, npacks), KOKKOS_LAMBDA (const int ilev) { const auto range_pack = ekat::range(ilev*N); diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 0f13141b9f6b..08dfe0ccb831 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -3,6 +3,8 @@ #include "dynamics/homme/physics_dynamics_remapper.hpp" #include "dynamics/homme/homme_dynamics_helpers.hpp" +#include "share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp" + #ifndef NDEBUG #include "share/property_checks/field_nan_check.hpp" #include "share/property_checks/field_lower_bound_check.hpp" @@ -137,7 +139,7 @@ void HommeGridsManager::build_dynamics_grid () { return; } - using gid_t = AbstractGrid::gid_type; + using gid_type = AbstractGrid::gid_type; // Get dimensions and create "empty" grid const int nlelem = get_num_local_elems_f90(); @@ -156,8 +158,8 @@ void HommeGridsManager::build_dynamics_grid () { auto lat = dyn_grid->create_geometry_data("lat",layout2d,rad); auto lon = dyn_grid->create_geometry_data("lon",layout2d,rad); - auto dg_dofs_h = dg_dofs.get_view(); - auto cg_dofs_h = cg_dofs.get_view(); + auto dg_dofs_h = dg_dofs.get_view(); + auto cg_dofs_h = cg_dofs.get_view(); auto elgpgp_h = elgpgp.get_view(); auto lat_h = lat.get_view(); auto lon_h = lon.get_view(); @@ -197,6 +199,10 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { return; } + if (type=="PG2") { + fvphyshack = true; + } + // Get the grid pg_type const int pg_code = m_pg_codes.at(type).at(rebalance); @@ -217,9 +223,9 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { auto lon = phys_grid->create_geometry_data("lon",layout2d,rad); auto area = phys_grid->create_geometry_data("area",layout2d,rad*rad); - using gid_t = AbstractGrid::gid_type; + using gid_type = AbstractGrid::gid_type; - auto dofs_h = dofs.get_view(); + auto dofs_h = dofs.get_view(); auto lat_h = lat.get_view(); auto lon_h = lon.get_view(); auto area_h = area.get_view(); @@ -264,6 +270,14 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { } } + if (is_planar_geometry_f90()) { + // If running with IOP, store grid length size + FieldLayout scalar0d({},{}); + auto dx_short_f = phys_grid->create_geometry_data("dx_short",scalar0d,rad); + dx_short_f.get_view()() = get_dx_short_f90(0); + dx_short_f.sync_to_dev(); + } + phys_grid->m_short_name = type; add_grid(phys_grid); } diff --git a/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 index 2faeb152a019..eefcd65e8d7f 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 @@ -65,11 +65,15 @@ end subroutine prim_init_data_structures_f90 subroutine prim_complete_init1_phase_f90 () bind(c) use prim_driver_base, only: prim_init1_buffers, prim_init1_compose, prim_init1_cleanup use homme_context_mod, only: par, elem +#ifdef HOMME_ENABLE_COMPOSE use compose_mod, only: compose_control_kokkos_init_and_fin +#endif use prim_driver_mod, only: prim_init_grid_views +#ifdef HOMME_ENABLE_COMPOSE ! Compose is not in charge of init/finalize kokkos call compose_control_kokkos_init_and_fin(.false.) +#endif ! Init compose call prim_init1_compose(par,elem) @@ -242,11 +246,16 @@ subroutine prim_run_f90 (nsplit_iteration) bind(c) end subroutine prim_run_f90 subroutine prim_finalize_f90 () bind(c) - use homme_context_mod, only: is_model_inited, elem, dom_mt, close_homme_log + use homme_context_mod, only: is_model_inited, elem, par, dom_mt, close_homme_log use prim_cxx_driver_base, only: prim_finalize if (.not. is_model_inited) then - call abortmp ("Error! prim_init_model_f90 was not called yet (or prim_finalize_f90 was already called).\n") + if (par%masterproc) then + print *, "WARNING! prim_init_model_f90 was not called yet (or prim_finalize_f90 was already called)" + print *, " We assume this is happening because an exception was thrown during initialization," + print *, " and we're destroying objects as part of the stack unwinding." + endif + return endif ! Cleanup some f90 stuff in Homme, and cleanup all cxx structures diff --git a/components/eamxx/src/dynamics/homme/interface/homme_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_grid_mod.F90 index 7c3100c53d4f..6f11024627f6 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_grid_mod.F90 @@ -16,9 +16,21 @@ module homme_grid_mod public :: get_num_local_columns_f90, get_num_global_columns_f90 public :: get_num_local_elems_f90, get_num_global_elems_f90 public :: get_np_f90, get_nlev_f90 + public :: is_planar_geometry_f90 contains + function is_planar_geometry_f90 () result(planar) bind(c) + use control_mod, only: geometry + logical (kind=c_bool) :: planar + + if (geometry == "plane") then + planar = .true. + else + planar = .false. + endif + end function is_planar_geometry_f90 + subroutine check_grids_inited (expected) use parallel_mod, only: abortmp use homme_context_mod, only: is_geometry_inited @@ -227,4 +239,19 @@ function get_nlev_f90 () result (nlev_out) bind(c) nlev_out = nlev end function get_nlev_f90 + function get_dx_short_f90 (elem_idx) result (dx_short_out) bind(c) + use homme_context_mod, only: elem + ! + ! Input(s) + ! + integer (kind=c_int), intent(in), value :: elem_idx + ! + ! Local(s) + ! + real (kind=c_double) :: dx_short_out + + ! elem_idx is 0-based, convert to 1-based + dx_short_out = elem(elem_idx+1)%dx_short + end function get_dx_short_f90 + end module homme_grid_mod diff --git a/components/eamxx/src/dynamics/homme/interface/homme_params_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_params_mod.F90 index 98a0881e5b00..9a3df59b6161 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_params_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_params_mod.F90 @@ -106,6 +106,7 @@ function get_homme_int_param_f90 (param_name_c) result(param_value) bind(c) param_value = dims(3) case default call abortmp ("[get_homme_int_param_f90] Error! Unrecognized parameter name.") + param_value = 0 end select end function get_homme_int_param_f90 @@ -143,6 +144,7 @@ function get_homme_real_param_f90 (param_name_c) result(param_value) bind(c) param_value = tstep case default call abortmp ("[get_homme_real_param_f90] Error! Unrecognized parameter name.") + param_value = 0 end select end function get_homme_real_param_f90 @@ -171,6 +173,7 @@ function get_homme_bool_param_f90 (param_name_c) result(param_value) bind(c) endif case default call abortmp ("[get_homme_bool_param_f90] Error! Unrecognized parameter name.") + param_value = .false. end select end function get_homme_bool_param_f90 diff --git a/components/eamxx/src/dynamics/homme/interface/scream_homme_interface.hpp b/components/eamxx/src/dynamics/homme/interface/scream_homme_interface.hpp index c6799a8bf340..8968fc49fb16 100644 --- a/components/eamxx/src/dynamics/homme/interface/scream_homme_interface.hpp +++ b/components/eamxx/src/dynamics/homme/interface/scream_homme_interface.hpp @@ -55,6 +55,7 @@ void prim_run_f90 (const int nsplit_iteration); void prim_finalize_f90 (); // Grids specs +bool is_planar_geometry_f90 (); int get_nlev_f90 (); int get_np_f90 (); int get_num_local_columns_f90 (const int pgN); @@ -69,6 +70,7 @@ void get_phys_grid_data_f90 (const int& pg_type, AbstractGrid::gid_type* const& gids, double* const& lat, double* const& lon, double* const& area); int get_homme_nsplit_f90 (const int& atm_dt); +double get_dx_short_f90 (const int elem_idx); // Parmaters getters/setters int get_homme_int_param_f90(const char** name); diff --git a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp index 5562980d3426..5ac1bb2eca6a 100644 --- a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp +++ b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp @@ -66,6 +66,10 @@ FieldLayout PhysicsDynamicsRemapper:: create_src_layout (const FieldLayout& tgt_layout) const { using namespace ShortFieldTagsNames; + EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), + "[PhysicsDynamicsRemapper] Error! Input target layout is not valid for this remapper.\n" + " - input layout: " + to_string(tgt_layout)); + auto tags = tgt_layout.tags(); auto dims = tgt_layout.dims(); @@ -96,6 +100,10 @@ FieldLayout PhysicsDynamicsRemapper:: create_tgt_layout (const FieldLayout& src_layout) const { using namespace ShortFieldTagsNames; + EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), + "[PhysicsDynamicsRemapper] Error! Input source layout is not valid for this remapper.\n" + " - input layout: " + to_string(src_layout)); + auto tags = src_layout.tags(); auto dims = src_layout.dims(); @@ -735,9 +743,9 @@ create_p2d_map () { auto se_dyn = std::dynamic_pointer_cast(m_dyn_grid); EKAT_REQUIRE_MSG(se_dyn, "Error! Something went wrong casting dyn grid to a SEGrid.\n"); - using gid_t = AbstractGrid::gid_type; - auto dyn_gids = se_dyn->get_cg_dofs_gids().get_view(); - auto phys_gids = m_phys_grid->get_dofs_gids().get_view(); + using gid_type = AbstractGrid::gid_type; + auto dyn_gids = se_dyn->get_cg_dofs_gids().get_view(); + auto phys_gids = m_phys_grid->get_dofs_gids().get_view(); auto policy = KokkosTypes::RangePolicy(0,num_phys_dofs); m_p2d = decltype(m_p2d) ("",num_phys_dofs); diff --git a/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt b/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt index d51a43fa677f..8b500fa239bc 100644 --- a/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt @@ -9,14 +9,14 @@ if (NOT SCREAM_BASELINES_ONLY) # Test dynamics-physics fields remapping CreateUnitTest(homme_pd_remap "homme_pd_remap_tests.cpp;test_helper_mod.F90" - "scream_share;${dynLibName}" + LIBS ${dynLibName} MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} - LABELS "dynamics") + LABELS dynamics) # Test I/O on dyn grid CreateUnitTest(dyn_grid_io "dyn_grid_io.cpp;test_helper_mod.F90" - "scream_share;scream_io;${dynLibName}" + LIBS scream_io ${dynLibName} MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} - LABELS "dynamics;io") + LABELS dynamics io) endif() diff --git a/components/eamxx/src/dynamics/homme/tests/homme_pd_remap_tests.cpp b/components/eamxx/src/dynamics/homme/tests/homme_pd_remap_tests.cpp index 97efb77c93b0..7d495ea0fb16 100644 --- a/components/eamxx/src/dynamics/homme/tests/homme_pd_remap_tests.cpp +++ b/components/eamxx/src/dynamics/homme/tests/homme_pd_remap_tests.cpp @@ -39,6 +39,7 @@ TEST_CASE("remap", "") { using IPDF = std::uniform_int_distribution; using FID = FieldIdentifier; using FL = FieldLayout; + using gid_type = AbstractGrid::gid_type; constexpr int pg_gll = 0; constexpr int PackSize = HOMMEXX_VECTOR_SIZE; @@ -81,8 +82,8 @@ TEST_CASE("remap", "") { // Get physics and dynamics grids, and their dofs auto phys_grid = gm.get_grid("Physics GLL"); auto dyn_grid = std::dynamic_pointer_cast(gm.get_grid("Dynamics")); - auto h_p_dofs = phys_grid->get_dofs_gids().get_view(); - auto h_d_dofs = dyn_grid->get_cg_dofs_gids().get_view(); + auto h_p_dofs = phys_grid->get_dofs_gids().get_view(); + auto h_d_dofs = dyn_grid->get_cg_dofs_gids().get_view(); auto h_d_lid2idx = dyn_grid->get_lid_to_idx_map().get_view(); // Get some dimensions for Homme @@ -573,6 +574,7 @@ TEST_CASE("combo_remap", "") { using IPDF = std::uniform_int_distribution; using FID = FieldIdentifier; using FL = FieldLayout; + using gid_type = AbstractGrid::gid_type; constexpr int pg_gll = 0; constexpr int PackSize = HOMMEXX_VECTOR_SIZE; @@ -615,8 +617,8 @@ TEST_CASE("combo_remap", "") { // Get physics and dynamics grids, and their dofs auto phys_grid = gm.get_grid("Physics GLL"); auto dyn_grid = std::dynamic_pointer_cast(gm.get_grid("Dynamics")); - auto h_p_dofs = phys_grid->get_dofs_gids().get_view(); - auto h_d_dofs = dyn_grid->get_cg_dofs_gids().get_view(); + auto h_p_dofs = phys_grid->get_dofs_gids().get_view(); + auto h_d_dofs = dyn_grid->get_cg_dofs_gids().get_view(); auto h_d_lid2idx = dyn_grid->get_lid_to_idx_map().get_view(); // Get some dimensions for Homme diff --git a/components/eamxx/src/dynamics/homme/tests/theta-dp.nl b/components/eamxx/src/dynamics/homme/tests/theta-dp.nl new file mode 100644 index 000000000000..b8555e513a57 --- /dev/null +++ b/components/eamxx/src/dynamics/homme/tests/theta-dp.nl @@ -0,0 +1,31 @@ +&ctl_nl +cubed_sphere_map = ${HOMME_TEST_CUBED_SPHERE_MAP} +disable_diagnostics = .false. +dt_remap_factor = ${HOMME_TEST_REMAP_FACTOR} +dt_tracer_factor = ${HOMME_TEST_TRACERS_FACTOR} +hypervis_order = 2 +hypervis_scaling = ${HOMME_TEST_HVSCALING} +hypervis_subcycle = ${HOMME_TEST_HVS} +hypervis_subcycle_tom = ${HOMME_TEST_HVS_TOM} +hypervis_subcycle_q = ${HOMME_TEST_HVS_Q} +mesh_file = "none" +nu = ${HOMME_TEST_NU} +nu_top = ${HOMME_TEST_NUTOP} +se_ftype = ${HOMME_SE_FTYPE} +se_geometry = "plane" +se_limiter_option = ${HOMME_TEST_LIM} +se_ne_x = ${HOMME_TEST_NE_X} +se_ne_y = ${HOMME_TEST_NE_Y} +se_lx = ${HOMME_TEST_LX} +se_ly = ${HOMME_TEST_LY} +se_nsplit = -1 +se_partmethod = 4 +se_topology = "plane" +se_tstep = ${HOMME_TEST_TIME_STEP} +statefreq = 9999 +theta_advect_form = ${HOMME_THETA_FORM} +theta_hydrostatic_mode = ${HOMME_THETA_HY_MODE} +transport_alg = ${HOMME_TEST_TRANSPORT_ALG} +tstep_type = ${HOMME_TTYPE} +vert_remap_q_alg = 10 +/ diff --git a/components/eamxx/src/dynamics/homme/tests/theta.nl b/components/eamxx/src/dynamics/homme/tests/theta.nl index a9f4628400b0..75ae06d2d008 100644 --- a/components/eamxx/src/dynamics/homme/tests/theta.nl +++ b/components/eamxx/src/dynamics/homme/tests/theta.nl @@ -1,5 +1,5 @@ &ctl_nl -cubed_sphere_map = 0 +cubed_sphere_map = ${HOMME_TEST_CUBED_SPHERE_MAP} disable_diagnostics = .false. dt_remap_factor = ${HOMME_TEST_REMAP_FACTOR} dt_tracer_factor = ${HOMME_TEST_TRACERS_FACTOR} @@ -7,6 +7,7 @@ hypervis_order = 2 hypervis_scaling = ${HOMME_TEST_HVSCALING} hypervis_subcycle = ${HOMME_TEST_HVS} hypervis_subcycle_tom = ${HOMME_TEST_HVS_TOM} +hypervis_subcycle_q = ${HOMME_TEST_HVS_Q} nu = ${HOMME_TEST_NU} nu_top = ${HOMME_TEST_NUTOP} se_ftype = ${HOMME_SE_FTYPE} @@ -20,6 +21,7 @@ se_tstep = ${HOMME_TEST_TIME_STEP} statefreq = 9999 theta_advect_form = ${HOMME_THETA_FORM} theta_hydrostatic_mode = ${HOMME_THETA_HY_MODE} +transport_alg = ${HOMME_TEST_TRANSPORT_ALG} tstep_type = ${HOMME_TTYPE} vert_remap_q_alg = 1 / diff --git a/components/eamxx/src/dynamics/register_dynamics.hpp b/components/eamxx/src/dynamics/register_dynamics.hpp index 9129ddebfc5c..da3c85cb21ee 100644 --- a/components/eamxx/src/dynamics/register_dynamics.hpp +++ b/components/eamxx/src/dynamics/register_dynamics.hpp @@ -3,8 +3,8 @@ #include "share/atm_process/atmosphere_process.hpp" -#ifdef SCREAM_HAS_HOMME -#include "homme/atmosphere_dynamics.hpp" +#ifdef EAMXX_HAS_HOMME +#include "homme/eamxx_homme_process_interface.hpp" #include "homme/homme_grids_manager.hpp" #endif @@ -14,7 +14,7 @@ inline void register_dynamics () { auto& proc_factory = AtmosphereProcessFactory::instance(); auto& gm_factory = GridsManagerFactory::instance(); -#ifdef SCREAM_HAS_HOMME +#ifdef EAMXX_HAS_HOMME proc_factory.register_product("Homme",&create_atmosphere_process); gm_factory.register_product("Homme",&create_homme_grids_manager); diff --git a/components/eamxx/src/mct_coupling/CMakeLists.txt b/components/eamxx/src/mct_coupling/CMakeLists.txt index a2bac4259370..308cd1776234 100644 --- a/components/eamxx/src/mct_coupling/CMakeLists.txt +++ b/components/eamxx/src/mct_coupling/CMakeLists.txt @@ -35,11 +35,19 @@ set (SCREAM_LIBS p3 shoc scream_rrtmgp + eamxx_cosp cld_fraction spa nudging diagnostics + tms ) +if (SCREAM_ENABLE_ML_CORRECTION) + list (APPEND SCREAM_LIBS ml_correction) +endif () +if (SCREAM_ENABLE_MAM) + list (APPEND SCREAM_LIBS mam) +endif() # Create atm lib add_library(atm ${ATM_SRC}) @@ -49,4 +57,4 @@ set_target_properties(atm PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR target_include_directories (atm PUBLIC ${CMAKE_BINARY_DIR}/cmake/atm) # Link libraries -target_link_libraries(atm PRIVATE ${SCREAM_LIBS} mct) +target_link_libraries(atm PRIVATE ${SCREAM_LIBS} csm_share) diff --git a/components/eamxx/src/mct_coupling/ScreamContext.hpp b/components/eamxx/src/mct_coupling/ScreamContext.hpp index 531a45db39f7..8b40134a0834 100644 --- a/components/eamxx/src/mct_coupling/ScreamContext.hpp +++ b/components/eamxx/src/mct_coupling/ScreamContext.hpp @@ -65,6 +65,10 @@ class ScreamContext std::map m_objects; }; +inline void cleanup_singleton() { + ScreamContext::singleton().clean_up(); +} + } // namespace scream #endif // SCREAM_CONTEXT_HPP diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index f38322be2989..34bbbedcc5c4 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -86,10 +86,13 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) character(CS) :: run_type ! type of run type(c_ptr) :: x2a_ptr, a2x_ptr character(len=256) :: atm_log_fname ! name of ATM log file + character(CL) :: calendar ! calendar string ! TODO: read this from the namelist? character(len=256) :: yaml_fname = "./data/scream_input.yaml" character(kind=c_char,len=256), target :: yaml_fname_c, atm_log_fname_c + character(len=256) :: caseid, username, hostname + character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c logical (kind=c_bool) :: restarted_run !------------------------------------------------------------------------------- @@ -101,7 +104,8 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) gsMap=gsmap_atm, & dom=dom_atm, & infodata=infodata) - call seq_infodata_getData(infodata, atm_phase=phase, start_type=run_type) + call seq_infodata_getData(infodata, atm_phase=phase, start_type=run_type, & + username=username, case_name=caseid, hostname=hostname) call seq_infodata_PutData(infodata, atm_aero=.true.) call seq_infodata_PutData(infodata, atm_prognostic=.true.) @@ -142,12 +146,16 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) !---------------------------------------------------------------------------- ! Init the AD - call seq_timemgr_EClockGetData(EClock, curr_ymd=cur_ymd, curr_tod=cur_tod, start_ymd=case_start_ymd, start_tod=case_start_tod) + call seq_timemgr_EClockGetData(EClock, calendar=calendar, & + curr_ymd=cur_ymd, curr_tod=cur_tod, & + start_ymd=case_start_ymd, start_tod=case_start_tod) call string_f2c(yaml_fname,yaml_fname_c) + call string_f2c(calendar,calendar_c) call string_f2c(trim(atm_log_fname),atm_log_fname_c) call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, & INT(cur_ymd,kind=C_INT), INT(cur_tod,kind=C_INT), & - INT(case_start_ymd,kind=C_INT), INT(case_start_tod,kind=C_INT)) + INT(case_start_ymd,kind=C_INT), INT(case_start_tod,kind=C_INT), & + calendar_c) ! Init MCT gsMap @@ -183,7 +191,10 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) c_loc(export_constant_multiple), c_loc(do_export_during_init), & num_cpl_exports, num_scream_exports, export_field_size) - call scream_init_atm () + call string_f2c(trim(caseid),caseid_c) + call string_f2c(trim(username),username_c) + call string_f2c(trim(hostname),hostname_c) + call scream_init_atm (caseid_c,hostname_c,username_c) end subroutine atm_init_mct diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 9b4d8875cdfd..0bdf90eeb71c 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -12,6 +12,7 @@ #include "mct_coupling/ScreamContext.hpp" #include "share/grid/point_grid.hpp" #include "share/scream_session.hpp" +#include "share/scream_config.hpp" #include "share/scream_types.hpp" #include "ekat/ekat_parse_yaml_file.hpp" @@ -51,7 +52,8 @@ void fpe_guard_wrapper (const Lambda& f) { // Execute wrapped function try { f(); - } catch (...) { + } catch (std::exception &e) { + fprintf(stderr, "%s\n", e.what()); auto& c = ScreamContext::singleton(); c.clean_up(); throw; @@ -85,7 +87,8 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, const int run_start_ymd, const int run_start_tod, const int case_start_ymd, - const int case_start_tod) + const int case_start_tod, + const char* calendar_name) { using namespace scream; using namespace scream::control; @@ -102,6 +105,17 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, // Initialize the scream session. scream::initialize_scream_session(atm_comm.am_i_root()); + std::string cal = calendar_name; + if (cal=="NO_LEAP") { + scream::set_use_leap_year (false); + } else if (cal=="GREGORIAN") { + scream::set_use_leap_year (true); + } else { + EKAT_ERROR_MSG ("Error! Invalid/unsupported calendar name.\n" + " - input name : " + cal + "\n" + " - valid names: NO_LEAP, GREGORIAN\n"); + } + // Create a parameter list for inputs ekat::ParameterList scream_params("Scream Parameters"); parse_yaml_file (input_yaml_file, scream_params); @@ -188,10 +202,9 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor }); } -void scream_init_atm (const int run_start_ymd, - const int run_start_tod, - const int case_start_ymd, - const int case_start_tod) +void scream_init_atm (const char* caseid, + const char* hostname, + const char* username) { using namespace scream; using namespace scream::control; @@ -200,9 +213,15 @@ void scream_init_atm (const int run_start_ymd, // Get the ad, then complete initialization auto& ad = get_ad_nonconst(); + // Set provenance info in the driver (will be added to the output files) + ad.set_provenance_data (caseid,hostname,username); + // Init all fields, atm processes, and output streams ad.initialize_fields (); ad.initialize_atm_procs (); + // Do this before init-ing the output managers, + // so the fields are valid if outputing at t=0 + ad.reset_accumulated_fields(); ad.initialize_output_managers (); }); } @@ -224,7 +243,11 @@ void scream_finalize (/* args ? */) { ad.finalize(); // Clean up the context - scream::ScreamContext::singleton().clean_up(); + // Note: doing the cleanup here via + // scream::ScreamContext::singleton().clean_up(); + // causes an ICE with C++17 on Summit/Ascent. + // Wrapping it in a function seems to work though + scream::cleanup_singleton(); // Finalize scream session scream::finalize_scream_session(); @@ -262,15 +285,15 @@ int scream_get_num_global_cols () { // Return the global ids of all physics column void scream_get_local_cols_gids (void* const ptr) { using namespace scream; - using gid_t = AbstractGrid::gid_type; + using gid_type = AbstractGrid::gid_type; fpe_guard_wrapper([&]() { - auto gids_f = reinterpret_cast(ptr); + auto gids_f = reinterpret_cast(ptr); const auto& ad = get_ad(); const auto& phys_grid = ad.get_grids_manager()->get_grid("Physics"); auto gids = phys_grid->get_dofs_gids(); gids.sync_to_host(); - auto gids_h = gids.get_view(); + auto gids_h = gids.get_view(); for (int i=0; i grids_mana // Define the different field layouts that will be used for this process - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; // Set of fields used strictly as input @@ -46,19 +46,19 @@ void CldFraction::set_grids(const std::shared_ptr grids_mana // Set of fields used strictly as output add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name,ps); add_field("cldfrac_ice", scalar3d_layout_mid, nondim, grid_name,ps); - // Note, we track two versions of the cloud fraction. The versions below have "_for_analysis" + // Note, we track two versions of the cloud fraction. The versions below have "_for_analysis" // attached to the name because they're meant for use with fields that are exclusively // related to writing output. This is an important distinction here because the internal ice - // cloud fraction needs to be 100% whenever any ice at all is present in the cell (in order - // for the model's ice processes to act on that cell). Folks evaluating cloud, on the other hand, - // expect cloud fraction to represent cloud visible to the human eye (which corresponds to - // ~1e-5 kg/kg). + // cloud fraction needs to be 100% whenever any ice at all is present in the cell (in order + // for the model's ice processes to act on that cell). Folks evaluating cloud, on the other hand, + // expect cloud fraction to represent cloud visible to the human eye (which corresponds to + // ~1e-5 kg/kg). add_field("cldfrac_tot_for_analysis", scalar3d_layout_mid, nondim, grid_name,ps); add_field("cldfrac_ice_for_analysis", scalar3d_layout_mid, nondim, grid_name,ps); // Set of fields used as input and output // - There are no fields used as both input and output. - + // Gather parameters for ice cloud thresholds from parameter list: m_icecloud_threshold = m_params.get("ice_cloud_threshold",1e-12); // Default = 1e-12 m_icecloud_for_analysis_threshold = m_params.get("ice_cloud_for_analysis_threshold",1e-5); // Default = 1e-5 @@ -79,7 +79,7 @@ void CldFraction::initialize_impl (const RunType /* run_type */) void CldFraction::run_impl (const double /* dt */) { // Calculate ice cloud fraction and total cloud fraction given the liquid cloud fraction - // and the ice mass mixing ratio. + // and the ice mass mixing ratio. auto qi = get_field_in("qi").get_view(); auto liq_cld_frac = get_field_in("cldfrac_liq").get_view(); auto ice_cld_frac = get_field_out("cldfrac_ice").get_view(); diff --git a/components/eamxx/src/physics/cld_fraction/atmosphere_cld_fraction.hpp b/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.hpp similarity index 100% rename from components/eamxx/src/physics/cld_fraction/atmosphere_cld_fraction.hpp rename to components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.hpp diff --git a/components/eamxx/src/physics/cosp/CMakeLists.txt b/components/eamxx/src/physics/cosp/CMakeLists.txt new file mode 100644 index 000000000000..4428ee504cdc --- /dev/null +++ b/components/eamxx/src/physics/cosp/CMakeLists.txt @@ -0,0 +1,55 @@ +# Build cosp external library and eamxx_cosp interface for EAMxx + +# Sources for EAMxx interface to COSP +set(COSP_SRCS + eamxx_cosp.cpp + cosp_c2f.F90 +) +set(COSP_HEADERS + eamxx_cosp.hpp + cosp_functions.hpp +) + +# Build external COSP library (this is all fortran code) +set(EAM_COSP_DIR ${SCREAM_BASE_DIR}/../eam/src/physics/cosp2) +set(EXTERNAL_SRC + ${EAM_COSP_DIR}/cosp_kinds.F90 + ${EAM_COSP_DIR}/external/src/cosp_constants.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_cloudsat_interface.F90 + ${EAM_COSP_DIR}/external/src/cosp_config.F90 + ${EAM_COSP_DIR}/local/cosp.F90 + ${EAM_COSP_DIR}/external/src/cosp_stats.F90 + ${EAM_COSP_DIR}/external/src/simulator/quickbeam/quickbeam.F90 + ${EAM_COSP_DIR}/external/src/simulator/parasol/parasol.F90 + ${EAM_COSP_DIR}/external/src/simulator/actsim/lidar_simulator.F90 + ${EAM_COSP_DIR}/external/src/simulator/icarus/icarus.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_atlid_interface.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_calipso_interface.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_grLidar532_interface.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_isccp_interface.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_misr_interface.F90 + ${EAM_COSP_DIR}/external/src/simulator/MISR_simulator/MISR_simulator.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_modis_interface.F90 + ${EAM_COSP_DIR}/external/src/simulator/MODIS_simulator/modis_simulator.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_rttov_interfaceSTUB.F90 + ${EAM_COSP_DIR}/external/src/simulator/rttov/cosp_rttovSTUB.F90 + ${EAM_COSP_DIR}/external/src/simulator/cosp_parasol_interface.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/subcol/scops.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/subcol/prec_scops.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/optics/cosp_utils.F90 #optics/cosp_utils.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/optics/cosp_optics.F90 #optics/cosp_optics.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/optics/quickbeam_optics/quickbeam_optics.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/subcol/rng/mo_rng.F90 + ${EAM_COSP_DIR}/cosp_errorHandling.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/optics/quickbeam_optics/array_lib.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/optics/quickbeam_optics/math_lib.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/optics/quickbeam_optics/optics_lib.F90 + ${EAM_COSP_DIR}/external/subsample_and_optics_example/optics/quickbeam_optics/mrgrnk.F90 +) +add_library(cosp ${EXTERNAL_SRC}) + +# Build interface code +add_library(eamxx_cosp ${COSP_SRCS}) +target_link_libraries(eamxx_cosp physics_share scream_share cosp) +target_compile_options(eamxx_cosp PUBLIC) +target_compile_definitions(eamxx_cosp PUBLIC EAMXX_HAS_COSP) diff --git a/components/eamxx/src/physics/cosp/cosp_c2f.F90 b/components/eamxx/src/physics/cosp/cosp_c2f.F90 new file mode 100644 index 000000000000..351207ae336d --- /dev/null +++ b/components/eamxx/src/physics/cosp/cosp_c2f.F90 @@ -0,0 +1,1350 @@ +! COSP interface to unwrap derived types for interoperability with C++ +module cosp_c2f + use iso_c_binding + use cosp_kinds, only: wp + use mod_cosp_config, only: R_UNDEF,PARASOL_NREFL,LIDAR_NCAT,LIDAR_NTYPE,SR_BINS, & + N_HYDRO,RTTOV_MAX_CHANNELS,numMISRHgtBins, & + cloudsat_DBZE_BINS,LIDAR_NTEMP,calipso_histBsct, & + CFODD_NDBZE, CFODD_NICOD, & + CFODD_BNDRE, CFODD_NCLASS, & + CFODD_DBZE_MIN, CFODD_DBZE_MAX, & + CFODD_ICOD_MIN, CFODD_ICOD_MAX, & + CFODD_DBZE_WIDTH, CFODD_ICOD_WIDTH, & + WR_NREGIME, & + numMODISTauBins,numMODISPresBins, & + numMODISReffIceBins,numMODISReffLiqBins, & + numISCCPTauBins,numISCCPPresBins,numMISRTauBins, & + ntau,modis_histTau,tau_binBounds, & + modis_histTauEdges,tau_binEdges, & + modis_histTauCenters,tau_binCenters,ntauV1p4, & + tau_binBoundsV1p4,tau_binEdgesV1p4, tau_binCentersV1p4, & + grLidar532_histBsct,atlid_histBsct,vgrid_zu,vgrid_zl, & + Nlvgrid, vgrid_z,cloudsat_preclvl + use cosp_phys_constants, only: amw,amd,amO3,amCO2,amCH4,amN2O,amCO + use mod_quickbeam_optics,only: size_distribution,hydro_class_init,quickbeam_optics, & + quickbeam_optics_init,gases + use quickbeam, only: radar_cfg + use mod_cosp, only: cosp_init,cosp_optical_inputs,cosp_column_inputs, & + cosp_outputs,cosp_cleanup,cosp_simulator + use mod_rng, only: rng_state, init_rng + use mod_scops, only: scops + use mod_prec_scops, only: prec_scops + use mod_cosp_utils, only: cosp_precip_mxratio + use cosp_optics, only: cosp_simulator_optics,lidar_optics,modis_optics, & + modis_optics_partition + use mod_cosp_stats, only: cosp_change_vertical_grid + + implicit none + + public :: cosp_c2f_init, cosp_c2f_run, cosp_c2f_final + + ! Local variables; control what runs and what does not + logical :: & + lsingle = .false., & ! True if using MMF_v3_single_moment CLOUDSAT microphysical scheme (default) + ldouble = .true., & ! True if using MMF_v3.5_two_moment CLOUDSAT microphysical scheme + lisccp = .true. , & ! Local on/off switch for simulators (used by initialization) + lmodis = .false., & ! + lmisr = .false., & ! + lcalipso = .false., & ! + lgrLidar532 = .false., & ! + latlid = .false., & ! + lcloudsat = .false., & ! + lrttov = .false., & ! + lparasol = .false. ! + + ! Logicals to control output + ! Hard-code logicals for now; these need to be consistent with EAMxx outputs anyways + logical :: & + Lpctisccp = .false., & ! ISCCP mean cloud top pressure + Lclisccp = .true. , & ! ISCCP cloud area fraction + Lboxptopisccp = .false., & ! ISCCP CTP in each column + Lboxtauisccp = .false., & ! ISCCP optical epth in each column + Ltauisccp = .false., & ! ISCCP mean optical depth + Lcltisccp = .true. , & ! ISCCP total cloud fraction + Lmeantbisccp = .false., & ! ISCCP mean all-sky 10.5micron brightness temperature + Lmeantbclrisccp = .false., & ! ISCCP mean clear-sky 10.5micron brightness temperature + Lalbisccp = .false., & ! ISCCP mean cloud albedo + LclMISR = .false., & ! MISR cloud fraction + Lcltmodis = .false., & ! MODIS total cloud fraction + Lclwmodis = .false., & ! MODIS liquid cloud fraction + Lclimodis = .false., & ! MODIS ice cloud fraction + Lclhmodis = .false., & ! MODIS high-level cloud fraction + Lclmmodis = .false., & ! MODIS mid-level cloud fraction + Lcllmodis = .false., & ! MODIS low-level cloud fraction + Ltautmodis = .false., & ! MODIS total cloud optical thicknes + Ltauwmodis = .false., & ! MODIS liquid optical thickness + Ltauimodis = .false., & ! MODIS ice optical thickness + Ltautlogmodis = .false., & ! MODIS total cloud optical thickness (log10 mean) + Ltauwlogmodis = .false., & ! MODIS liquid optical thickness (log10 mean) + Ltauilogmodis = .false., & ! MODIS ice optical thickness (log10 mean) + Lreffclwmodis = .false., & ! MODIS liquid cloud particle size + Lreffclimodis = .false., & ! MODIS ice particle size + Lpctmodis = .false., & ! MODIS cloud top pressure + Llwpmodis = .false., & ! MODIS cloud liquid water path + Liwpmodis = .false., & ! MODIS cloud ice water path + Lclmodis = .false., & ! MODIS cloud area fraction + Latb532 = .false., & ! CALIPSO attenuated total backscatter (532nm) + Latb532gr = .false., & ! GROUND LIDAR @ 532NM attenuated total backscatter (532nm) + Latb355 = .false., & ! ATLID attenuated total backscatter (355nm) + LlidarBetaMol532 = .false., & ! CALIPSO molecular backscatter (532nm) + LlidarBetaMol532gr = .false., & ! GROUND LIDAR @ 532NM molecular backscatter (532nm) + LlidarBetaMol355 = .false., & ! ATLID molecular backscatter (355nm) + LcfadLidarsr532 = .false., & ! CALIPSO scattering ratio CFAD + LcfadLidarsr532gr = .false., & ! GROUND LIDAR @ 532NM scattering ratio CFAD + LcfadLidarsr355 = .false., & ! ATLID scattering ratio CFAD + Lclcalipso2 = .false., & ! CALIPSO cloud fraction undetected by cloudsat + Lclcalipso = .false., & ! CALIPSO cloud area fraction + LclgrLidar532 = .false., & ! GROUND LIDAR @ 532NM cloud area fraction + Lclatlid = .false., & ! ATLID cloud area fraction + Lclhcalipso = .false., & ! CALIPSO high-level cloud fraction + Lcllcalipso = .false., & ! CALIPSO low-level cloud fraction + Lclmcalipso = .false., & ! CALIPSO mid-level cloud fraction + Lcltcalipso = .false., & ! CALIPSO total cloud fraction + LclhgrLidar532 = .false., & ! GROUND LIDAR @ 532NM high-level cloud fraction + LcllgrLidar532 = .false., & ! GROUND LIDAR @ 532NM low-level cloud fraction + LclmgrLidar532 = .false., & ! GROUND LIDAR @ 532NM mid-level cloud fraction + LcltgrLidar532 = .false., & ! GROUND LIDAR @ 532NM total cloud fraction + Lclhatlid = .false., & ! ATLID high-level cloud fraction + Lcllatlid = .false., & ! ATLID low-level cloud fraction + Lclmatlid = .false., & ! ATLID mid-level cloud fraction + Lcltatlid = .false., & ! ATLID total cloud fraction + Lcltlidarradar = .false., & ! CALIPSO-CLOUDSAT total cloud fraction + Lcloudsat_tcc = .false., & ! + Lcloudsat_tcc2 = .false., & ! + Lclcalipsoliq = .false., & ! CALIPSO liquid cloud area fraction + Lclcalipsoice = .false., & ! CALIPSO ice cloud area fraction + Lclcalipsoun = .false., & ! CALIPSO undetected cloud area fraction + Lclcalipsotmp = .false., & ! CALIPSO undetected cloud area fraction + Lclcalipsotmpliq = .false., & ! CALIPSO liquid cloud area fraction + Lclcalipsotmpice = .false., & ! CALIPSO ice cloud area fraction + Lclcalipsotmpun = .false., & ! CALIPSO undetected cloud area fraction + Lcltcalipsoliq = .false., & ! CALIPSO liquid total cloud fraction + Lcltcalipsoice = .false., & ! CALIPSO ice total cloud fraction + Lcltcalipsoun = .false., & ! CALIPSO undetected total cloud fraction + Lclhcalipsoliq = .false., & ! CALIPSO high-level liquid cloud fraction + Lclhcalipsoice = .false., & ! CALIPSO high-level ice cloud fraction + Lclhcalipsoun = .false., & ! CALIPSO high-level undetected cloud fraction + Lclmcalipsoliq = .false., & ! CALIPSO mid-level liquid cloud fraction + Lclmcalipsoice = .false., & ! CALIPSO mid-level ice cloud fraction + Lclmcalipsoun = .false., & ! CALIPSO mid-level undetected cloud fraction + Lcllcalipsoliq = .false., & ! CALIPSO low-level liquid cloud fraction + Lcllcalipsoice = .false., & ! CALIPSO low-level ice cloud fraction + Lcllcalipsoun = .false., & ! CALIPSO low-level undetected cloud fraction + Lclopaquecalipso = .false., & ! CALIPSO opaque cloud cover (2D Map) + Lclthincalipso = .false., & ! CALIPSO thin cloud cover (2D Map) + Lclzopaquecalipso = .false., & ! CALIPSO z_opaque altitude (opaque clouds only, 2D Map) + Lclcalipsoopaque = .false., & ! CALIPSO opaque cloud profiles 3D fraction + Lclcalipsothin = .false., & ! CALIPSO thin cloud profiles 3D fraction + Lclcalipsozopaque = .false., & ! CALIPSO z_opaque 3D fraction + Lclcalipsoopacity = .false., & ! CALIPSO opacity 3D fraction + Lclopaquetemp = .false., & ! CALIPSO opaque cloud temperature + Lclthintemp = .false., & ! CALIPSO thin cloud temperature + Lclzopaquetemp = .false., & ! CALIPSO z_opaque temperature + Lclopaquemeanz = .false., & ! CALIPSO opaque cloud altitude + Lclthinmeanz = .false., & ! CALIPSO thin cloud altitude + Lclthinemis = .false., & ! CALIPSO thin cloud emissivity + Lclopaquemeanzse = .false., & ! CALIPSO opaque cloud altitude with respect to SE + Lclthinmeanzse = .false., & ! CALIPSO thin cloud altitude with respect to SE + Lclzopaquecalipsose = .false., & ! CALIPSO z_opaque altitude with respect to SE + LcfadDbze94 = .false., & ! CLOUDSAT radar reflectivity CFAD + Ldbze94 = .false., & ! CLOUDSAT radar reflectivity + LparasolRefl = .false., & ! PARASOL reflectance + Ltbrttov = .false., & ! RTTOV mean clear-sky brightness temperature + Lptradarflag0 = .false., & ! CLOUDSAT + Lptradarflag1 = .false., & ! CLOUDSAT + Lptradarflag2 = .false., & ! CLOUDSAT + Lptradarflag3 = .false., & ! CLOUDSAT + Lptradarflag4 = .false., & ! CLOUDSAT + Lptradarflag5 = .false., & ! CLOUDSAT + Lptradarflag6 = .false., & ! CLOUDSAT + Lptradarflag7 = .false., & ! CLOUDSAT + Lptradarflag8 = .false., & ! CLOUDSAT + Lptradarflag9 = .false., & ! CLOUDSAT + Lradarpia = .false., & ! CLOUDSAT + Lwr_occfreq = .false., & ! CloudSat+MODIS joint diagnostics + Lcfodd = .false. ! CloudSat+MODIS joint diagnostics + + ! Input namelist fields (hard-code these) + integer, parameter :: rttov_Nchannels = 3 + real(wp), dimension(rttov_Nchannels) :: rttov_surfem = (/0.0, 0.0, 0.0/) + integer :: & ! + !Nlvgrid = 40, & ! Number of vertical levels for statistical outputs (USE_VGRID=.true.) + surface_radar = 0, & ! surface=1/spaceborne=0 + cloudsat_use_gas_abs = 1, & ! Include gaseous absorption (1=yes/0=no) + cloudsat_do_ray = 0, & ! Calculate output Rayleigh (1=yes/0=no) + lidar_ice_type = 0, & ! Ice particle shape in lidar calculations (0=ice-spheres/1=ice-non-spherical) + overlap = 3, & ! Overlap type: 1=max, 2=rand, 3=max/rand + isccp_topheight = 1, & ! ISCCP cloud top height + isccp_topheight_direction = 2, & ! ISCCP cloud top height direction + rttov_platform = 1, & ! RTTOV: Satellite platform + rttov_satellite = 15, & ! RTTOV: Satellite + rttov_instrument = 5, & ! RTTOV: Instrument + rttov_channels(rttov_Nchannels) = (/1, 2, 3/) ! RTTOV: Number of channels to be computed + real(wp) :: & ! + cloudsat_radar_freq = 94.0, & ! CloudSat radar frequency (GHz) + cloudsat_k2 = -1, & ! |K|^2, -1=use frequency dependent default + rttov_ZenAng = 50.0, & ! RTTOV: Satellite Zenith Angle + co2 = 5.241e-04, & ! CO2 mixing ratio + ch4 = 9.139e-07, & ! CH4 mixing ratio + n2o = 4.665e-07, & ! n2o mixing ratio + co = 2.098e-07 ! co mixing ratio + logical :: & ! + use_vgrid = .true., & ! Use fixed vertical grid for outputs? + csat_vgrid = .true. ! CloudSat vertical grid? + + ! These only need to be allocated once, so we let them persist as module data + type(size_distribution) :: sd + type(radar_cfg) :: rcfg_cloudsat + type(cosp_outputs) :: cospOUT + type(cosp_optical_inputs) :: cospIN + type(cosp_column_inputs) :: cospstateIn + + + ! Indices to address arrays of LS and CONV hydrometeors + integer,parameter :: & + I_LSCLIQ = 1, & ! Large-scale (stratiform) liquid + I_LSCICE = 2, & ! Large-scale (stratiform) ice + I_LSRAIN = 3, & ! Large-scale (stratiform) rain + I_LSSNOW = 4, & ! Large-scale (stratiform) snow + I_CVCLIQ = 5, & ! Convective liquid + I_CVCICE = 6, & ! Convective ice + I_CVRAIN = 7, & ! Convective rain + I_CVSNOW = 8, & ! Convective snow + I_LSGRPL = 9 ! Large-scale (stratiform) groupel + + ! Stratiform and convective clouds in frac_out (scops output). + integer, parameter :: & + I_LSC = 1, & ! Large-scale clouds + I_CVC = 2 ! Convective clouds + + ! Microphysical settings for the precipitation flux to mixing ratio conversion + real(wp),parameter,dimension(N_HYDRO) :: & + ! LSL LSI LSR LSS CVL CVI CVR CVS LSG + N_ax = (/-1., -1., 8.e6, 3.e6, -1., -1., 8.e6, 3.e6, 4.e6/),& + N_bx = (/-1., -1., 0.0, 0.0, -1., -1., 0.0, 0.0, 0.0/),& + alpha_x = (/-1., -1., 0.0, 0.0, -1., -1., 0.0, 0.0, 0.0/),& + c_x = (/-1., -1., 842.0, 4.84, -1., -1., 842.0, 4.84, 94.5/),& + d_x = (/-1., -1., 0.8, 0.25, -1., -1., 0.8, 0.25, 0.5/),& + g_x = (/-1., -1., 0.5, 0.5, -1., -1., 0.5, 0.5, 0.5/),& + a_x = (/-1., -1., 524.0, 52.36, -1., -1., 524.0, 52.36, 209.44/),& + b_x = (/-1., -1., 3.0, 3.0, -1., -1., 3.0, 3.0, 3.0/),& + gamma_1 = (/-1., -1., 17.83725, 8.284701, -1., -1., 17.83725, 8.284701, 11.63230/),& + gamma_2 = (/-1., -1., 6.0, 6.0, -1., -1., 6.0, 6.0, 6.0/),& + gamma_3 = (/-1., -1., 2.0, 2.0, -1., -1., 2.0, 2.0, 2.0/),& + gamma_4 = (/-1., -1., 6.0, 6.0, -1., -1., 6.0, 6.0, 6.0/) + + character(len=64) :: cloudsat_micro_scheme = 'MMF_v3.5_two_moment' + +contains + + subroutine cosp_c2f_init(npoints, ncolumns, nlevels) bind(c, name='cosp_c2f_init') + integer(kind=c_int), value, intent(in) :: npoints, ncolumns, nlevels + ! Initialize/allocate COSP input and output derived types + nlvgrid = 40 + call construct_cospIN(npoints,ncolumns,nlevels,cospIN) + call construct_cospstatein(npoints,nlevels,rttov_nchannels,cospstateIN) + call construct_cosp_outputs(npoints, ncolumns, nlevels, nlvgrid, rttov_nchannels, cospOUT) + + ! Initialize quickbeam_optics, also if two-moment radar microphysics scheme is wanted... + if (cloudsat_micro_scheme == 'MMF_v3.5_two_moment') then + ldouble = .true. + lsingle = .false. + endif + + if (Lcloudsat) then + call quickbeam_optics_init() + + ! Initialize the distributional parameters for hydrometeors in radar simulator + call hydro_class_init(lsingle,ldouble,sd) + end if + + ! Initialize COSP simulator + call cosp_init(Lisccp, Lmodis, Lmisr, Lcloudsat, Lcalipso, LgrLidar532, Latlid, & + Lparasol, Lrttov, & + cloudsat_radar_freq, cloudsat_k2, cloudsat_use_gas_abs, & + cloudsat_do_ray, isccp_topheight, isccp_topheight_direction, surface_radar, & + rcfg_cloudsat, use_vgrid, csat_vgrid, Nlvgrid, Nlevels, cloudsat_micro_scheme) + end subroutine cosp_c2f_init + + subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & + emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, qv, & + cldfrac, reff_qc, reff_qi, dtau067, dtau105, isccp_cldtot, isccp_ctptau & + ) bind(C, name='cosp_c2f_run') + integer(kind=c_int), value, intent(in) :: npoints, ncolumns, nlevels, ntau, nctp + real(kind=c_double), value, intent(in) :: emsfc_lw + real(kind=c_double), intent(in), dimension(npoints) :: sunlit, skt + real(kind=c_double), intent(in), dimension(npoints,nlevels) :: T_mid, p_mid, qv, cldfrac, reff_qc, reff_qi, dtau067, dtau105 + real(kind=c_double), intent(in), dimension(npoints,nlevels+1) :: p_int + real(kind=c_double), intent(inout), dimension(npoints) :: isccp_cldtot + real(kind=c_double), intent(inout), dimension(npoints,ntau,nctp) :: isccp_ctptau + ! Takes normal arrays as input and populates COSP derived types + character(len=256),dimension(100) :: cosp_status + integer :: nptsperit + integer :: start_idx + integer :: end_idx + + ! Locals for subsample_and_optics + ! TODO: trim this down + real(wp), dimension(nPoints,nLevels) :: tca,cca,mr_lsliq,mr_lsice,mr_ccliq, & + mr_ccice,dtau_c,dtau_s,dem_c,dem_s,mr_lsrain,mr_lssnow,mr_lsgrpl,mr_ccrain,& + mr_ccsnow + real(wp), dimension(nPoints,nLevels,N_HYDRO) :: reff + + nptsperit = npoints + + tca(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) + cca(:npoints,:nlevels) = 0 + mr_lsliq(:npoints,:nlevels) = 0 + mr_ccliq(:npoints,:nlevels) = 0 + mr_lsice(:npoints,:nlevels) = 0 + mr_ccice(:npoints,:nlevels) = 0 + dtau_c(:npoints,:nlevels) = 0 + dtau_s(:npoints,:nlevels) = dtau067(:npoints,:nlevels) + dem_c (:npoints,:nlevels) = 0 + dem_s (:npoints,:nlevels) = 1._wp - exp(-dtau105(:npoints,:nlevels)) + mr_lsrain(:npoints,:nlevels) = 0 + mr_ccrain(:npoints,:nlevels) = 0 + mr_lssnow(:npoints,:nlevels) = 0 + mr_lssnow(:npoints,:nlevels) = 0 + mr_ccsnow(:npoints,:nlevels) = 0 + mr_lsgrpl(:npoints,:nlevels) = 0 + reff = 0 ! FIXME + + start_idx = 1 + end_idx = npoints + ! Translate arrays to derived types + cospIN%emsfc_lw = emsfc_lw + cospIN%rcfg_cloudsat = rcfg_cloudsat +! cospstateIN%hgt_matrix = zlev(start_idx:end_idx,Nlevels:1:-1) ! km + cospstateIN%sunlit = sunlit(start_idx:end_idx) ! 0-1 + cospstateIN%skt = skt(start_idx:end_idx) ! K +! cospstateIN%surfelev = surfelev(start_idx:end_idx) ! m +! cospstateIN%land = landmask(start_idx:end_idx) ! 0-1 (*note* model specific) + cospstateIN%qv = qv(start_idx:end_idx,1:Nlevels) ! kg/kg + cospstateIN%at = T_mid(start_idx:end_idx,1:Nlevels) !Nlevels:1:-1) ! K + cospstateIN%pfull = p_mid(start_idx:end_idx,1:Nlevels) !Nlevels:1:-1) ! Pa + ! Pressure at interface (nlevels+1). Set uppermost interface to 0. + !cospstateIN%phalf(:,2:Nlevels+1) = p_int(start_idx:end_idx,Nlevels:1:-1) ! Pa + cospstateIN%phalf(:,1:Nlevels+1) = p_int(start_idx:end_idx,1:Nlevels+1) ! Pa +! ! Height of bottom interfaces of model layers (nlevels). +! ! cospstateIN%hgt_matrix_half(:,1) contains the bottom of the top layer. +! ! cospstateIN%hgt_matrix_half(:,Nlevels) contains the bottom of the surface layer. +! cospstateIN%hgt_matrix_half(:,1:Nlevels) = zlev_half(start_idx:end_idx,Nlevels:1:-1) ! km + + ! Generate subcolumns and compute optical inputs. + call subsample_and_optics(nPoints, nLevels, nColumns, N_HYDRO, overlap, use_vgrid, & + lidar_ice_type, sd, tca, cca, mr_lsrain, mr_lssnow, & + mr_lsgrpl, mr_ccrain, mr_ccsnow, mr_lsliq, mr_lsice, mr_ccliq, mr_ccice, & + reff, dtau_c, dtau_s, dem_c, dem_s, cospstateIN, cospIN) + + ! Call cosp + cosp_status = cosp_simulator(cospIN, cospstateIN, cospOUT, start_idx, end_idx, .false.) + + ! Translate derived types to output arrays + isccp_cldtot(:npoints) = cospOUT%isccp_totalcldarea(:npoints) + isccp_ctptau(:npoints,:,:) = cospOUT%isccp_fq(:npoints,:,:) + + end subroutine cosp_c2f_run + + subroutine cosp_c2f_final() bind(C, name='cosp_c2f_final') + call destroy_cospIN(cospIN) + call destroy_cospstateIN(cospstateIN) + call destroy_cosp_outputs(cospOUT) + end subroutine cosp_c2f_final + + ! These are mostly copied from cosp_test.f90, and are pretty obnoxious + ! TODO: clean this up! + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! SUBROUTINE construct_cospIN + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + subroutine construct_cospIN(npoints,ncolumns,nlevels,y) + ! Inputs + integer,intent(in) :: & + npoints, & ! Number of horizontal gridpoints + ncolumns, & ! Number of subcolumns + nlevels ! Number of vertical levels + ! Outputs + type(cosp_optical_inputs),intent(out) :: y + + ! Dimensions + y%Npoints = Npoints + y%Ncolumns = Ncolumns + y%Nlevels = Nlevels + y%Npart = 4 + y%Nrefl = PARASOL_NREFL + allocate(y%frac_out(npoints, ncolumns,nlevels)) + + if (Lmodis .or. Lmisr .or. Lisccp) then + allocate(y%tau_067(npoints, ncolumns,nlevels),& + y%emiss_11(npoints, ncolumns,nlevels)) + endif + if (Lcalipso) then + allocate(y%betatot_calipso(npoints, ncolumns,nlevels),& + y%betatot_ice_calipso(npoints, ncolumns,nlevels),& + y%betatot_liq_calipso(npoints, ncolumns,nlevels),& + y%tautot_calipso(npoints, ncolumns,nlevels),& + y%tautot_ice_calipso(npoints, ncolumns,nlevels),& + y%tautot_liq_calipso(npoints, ncolumns,nlevels),& + y%beta_mol_calipso(npoints, nlevels),& + y%tau_mol_calipso(npoints, nlevels),& + y%tautot_S_ice(npoints, ncolumns ),& + y%tautot_S_liq(npoints, ncolumns )) + endif + + if (LgrLidar532) then + allocate(y%beta_mol_grLidar532(npoints, nlevels),& + y%betatot_grLidar532(npoints, ncolumns,nlevels),& + y%tau_mol_grLidar532(npoints, nlevels),& + y%tautot_grLidar532(npoints, ncolumns,nlevels)) + endif + + if (Latlid) then + allocate(y%beta_mol_atlid(npoints, nlevels),& + y%betatot_atlid(npoints, ncolumns,nlevels),& + y%tau_mol_atlid(npoints, nlevels),& + y%tautot_atlid(npoints, ncolumns,nlevels)) + endif + + if (Lcloudsat) then + allocate(y%z_vol_cloudsat(npoints, ncolumns,nlevels),& + y%kr_vol_cloudsat(npoints, ncolumns,nlevels),& + y%g_vol_cloudsat(npoints, ncolumns,nlevels),& + y%fracPrecipIce(npoints, ncolumns)) + endif + if (Lmodis) then + allocate(y%fracLiq(npoints, ncolumns,nlevels),& + y%asym(npoints, ncolumns,nlevels),& + y%ss_alb(npoints, ncolumns,nlevels)) + endif + + + end subroutine construct_cospIN + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! SUBROUTINE construct_cospstateIN + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + subroutine construct_cospstateIN(npoints,nlevels,nchan,y) + ! Inputs + integer,intent(in) :: & + npoints, & ! Number of horizontal gridpoints + nlevels, & ! Number of vertical levels + nchan ! Number of channels + ! Outputs + type(cosp_column_inputs),intent(out) :: y + + allocate(y%sunlit(npoints),y%skt(npoints),y%land(npoints),y%at(npoints,nlevels), & + y%pfull(npoints,nlevels),y%phalf(npoints,nlevels+1),y%qv(npoints,nlevels), & + y%o3(npoints,nlevels),y%hgt_matrix(npoints,nlevels),y%u_sfc(npoints), & + y%v_sfc(npoints),y%lat(npoints),y%lon(nPoints),y%emis_sfc(nchan), & + y%cloudIce(nPoints,nLevels),y%cloudLiq(nPoints,nLevels),y%surfelev(npoints),& + y%fl_snow(nPoints,nLevels),y%fl_rain(nPoints,nLevels),y%seaice(npoints), & + y%tca(nPoints,nLevels),y%hgt_matrix_half(npoints,nlevels)) + + end subroutine construct_cospstateIN + + ! ###################################################################################### + ! SUBROUTINE construct_cosp_outputs + ! + ! This subroutine allocates output fields based on input logical flag switches. + ! ###################################################################################### + ! TODO: This is WAY too many dummy arguments! These can just be defined at module scope I think + ! and then initialized once at init + subroutine construct_cosp_outputs(Npoints,Ncolumns,Nlevels,Nlvgrid,Nchan,x) + ! Inputs + integer,intent(in) :: & + Npoints, & ! Number of sampled points + Ncolumns, & ! Number of subgrid columns + Nlevels, & ! Number of model levels + Nlvgrid, & ! Number of levels in L3 stats computation + Nchan ! Number of RTTOV channels + + ! Outputs + type(cosp_outputs),intent(out) :: x ! COSP output structure + + ! ISCCP simulator outputs + if (Lboxtauisccp) allocate(x%isccp_boxtau(Npoints,Ncolumns)) + if (Lboxptopisccp) allocate(x%isccp_boxptop(Npoints,Ncolumns)) + if (Lclisccp) allocate(x%isccp_fq(Npoints,numISCCPTauBins,numISCCPPresBins)) + if (Lcltisccp) allocate(x%isccp_totalcldarea(Npoints)) + if (Lpctisccp) allocate(x%isccp_meanptop(Npoints)) + if (Ltauisccp) allocate(x%isccp_meantaucld(Npoints)) + if (Lmeantbisccp) allocate(x%isccp_meantb(Npoints)) + if (Lmeantbclrisccp) allocate(x%isccp_meantbclr(Npoints)) + if (Lalbisccp) allocate(x%isccp_meanalbedocld(Npoints)) + + ! MISR simulator + if (LclMISR) then + allocate(x%misr_fq(Npoints,numMISRTauBins,numMISRHgtBins)) + ! *NOTE* These 3 fields are not output, but were part of the v1.4.0 cosp_misr, so + ! they are still computed. Should probably have a logical to control these + ! outputs. + allocate(x%misr_dist_model_layertops(Npoints,numMISRHgtBins)) + allocate(x%misr_meanztop(Npoints)) + allocate(x%misr_cldarea(Npoints)) + endif + + ! MODIS simulator + if (Lcltmodis) allocate(x%modis_Cloud_Fraction_Total_Mean(Npoints)) + if (Lclwmodis) allocate(x%modis_Cloud_Fraction_Water_Mean(Npoints)) + if (Lclimodis) allocate(x%modis_Cloud_Fraction_Ice_Mean(Npoints)) + if (Lclhmodis) allocate(x%modis_Cloud_Fraction_High_Mean(Npoints)) + if (Lclmmodis) allocate(x%modis_Cloud_Fraction_Mid_Mean(Npoints)) + if (Lcllmodis) allocate(x%modis_Cloud_Fraction_Low_Mean(Npoints)) + if (Ltautmodis) allocate(x%modis_Optical_Thickness_Total_Mean(Npoints)) + if (Ltauwmodis) allocate(x%modis_Optical_Thickness_Water_Mean(Npoints)) + if (Ltauimodis) allocate(x%modis_Optical_Thickness_Ice_Mean(Npoints)) + if (Ltautlogmodis) allocate(x%modis_Optical_Thickness_Total_LogMean(Npoints)) + if (Ltauwlogmodis) allocate(x%modis_Optical_Thickness_Water_LogMean(Npoints)) + if (Ltauilogmodis) allocate(x%modis_Optical_Thickness_Ice_LogMean(Npoints)) + if (Lreffclwmodis) allocate(x%modis_Cloud_Particle_Size_Water_Mean(Npoints)) + if (Lreffclimodis) allocate(x%modis_Cloud_Particle_Size_Ice_Mean(Npoints)) + if (Lpctmodis) allocate(x%modis_Cloud_Top_Pressure_Total_Mean(Npoints)) + if (Llwpmodis) allocate(x%modis_Liquid_Water_Path_Mean(Npoints)) + if (Liwpmodis) allocate(x%modis_Ice_Water_Path_Mean(Npoints)) + if (Lclmodis) then + allocate(x%modis_Optical_Thickness_vs_Cloud_Top_Pressure(nPoints,numModisTauBins,numMODISPresBins)) + allocate(x%modis_Optical_thickness_vs_ReffLIQ(nPoints,numMODISTauBins,numMODISReffLiqBins)) + allocate(x%modis_Optical_Thickness_vs_ReffICE(nPoints,numMODISTauBins,numMODISReffIceBins)) + endif + + ! LIDAR simulator + if (LlidarBetaMol532) allocate(x%calipso_beta_mol(Npoints,Nlevels)) + if (Latb532) allocate(x%calipso_beta_tot(Npoints,Ncolumns,Nlevels)) + if (LcfadLidarsr532) then + allocate(x%calipso_srbval(SR_BINS+1)) + allocate(x%calipso_cfad_sr(Npoints,SR_BINS,Nlvgrid)) + allocate(x%calipso_betaperp_tot(Npoints,Ncolumns,Nlevels)) + endif + if (Lclcalipso) allocate(x%calipso_lidarcld(Npoints,Nlvgrid)) + if (Lclhcalipso .or. Lclmcalipso .or. Lcllcalipso .or. Lcltcalipso) then + allocate(x%calipso_cldlayer(Npoints,LIDAR_NCAT)) + endif + if (Lclcalipsoice .or. Lclcalipsoliq .or. Lclcalipsoun) then + allocate(x%calipso_lidarcldphase(Npoints,Nlvgrid,6)) + endif + if (Lclcalipsotmp .or. Lclcalipsotmpliq .or. Lclcalipsoice .or. Lclcalipsotmpun .or. Lclcalipsotmpice) then + allocate(x%calipso_lidarcldtmp(Npoints,LIDAR_NTEMP,5)) + endif + if (Lcllcalipsoice .or. Lclmcalipsoice .or. Lclhcalipsoice .or. & + Lcltcalipsoice .or. Lcllcalipsoliq .or. Lclmcalipsoliq .or. & + Lclhcalipsoliq .or. Lcltcalipsoliq .or. Lcllcalipsoun .or. & + Lclmcalipsoun .or. Lclhcalipsoun .or. Lcltcalipsoun) then + allocate(x%calipso_cldlayerphase(Npoints,LIDAR_NCAT,6)) + endif + if (Lclopaquecalipso .or. Lclthincalipso .or. Lclzopaquecalipso) then + allocate(x%calipso_cldtype(Npoints,LIDAR_NTYPE)) + endif + if (Lclopaquetemp .or. Lclthintemp .or. Lclzopaquetemp) then + allocate(x%calipso_cldtypetemp(Npoints,LIDAR_NTYPE)) + endif + if (Lclopaquemeanz .or. Lclthinmeanz) then + allocate(x%calipso_cldtypemeanz(Npoints,2)) + endif + if (Lclopaquemeanzse .or. Lclthinmeanzse .or. Lclzopaquecalipsose) then + allocate(x%calipso_cldtypemeanzse(Npoints,3)) + endif + if (Lclthinemis) then + allocate(x%calipso_cldthinemis(Npoints)) + endif + if (Lclcalipsoopaque .or. Lclcalipsothin .or. Lclcalipsozopaque .or. Lclcalipsoopacity) then + allocate(x%calipso_lidarcldtype(Npoints,Nlvgrid,LIDAR_NTYPE+1)) + endif + ! These 2 outputs are part of the calipso output type, but are not controlled by an + ! logical switch in the output namelist, so if all other fields are on, then allocate + if (LlidarBetaMol532 .or. Latb532 .or. LcfadLidarsr532 .or. Lclcalipso .or. & + Lclcalipsoice .or. Lclcalipsoliq .or. Lclcalipsoun .or. Lclcalipso2 .or. & + Lclhcalipso .or. Lclmcalipso .or. Lcllcalipso .or. Lcltcalipso .or. & + Lclcalipsotmp .or. Lclcalipsoice .or. Lclcalipsotmpun .or. & + Lclcalipsotmpliq .or. Lcllcalipsoice .or. Lclmcalipsoice .or. & + Lclhcalipsoice .or. Lcltcalipsoice .or. Lcllcalipsoliq .or. & + Lclmcalipsoliq .or. Lclhcalipsoliq .or. Lcltcalipsoliq .or. & + Lcllcalipsoun .or. Lclmcalipsoun .or. Lclhcalipsoun .or. Lcltcalipsoun) then + allocate(x%calipso_tau_tot(Npoints,Ncolumns,Nlevels)) + allocate(x%calipso_temp_tot(Npoints,Nlevels)) + endif + + ! GROUND LIDAR @ 532NM simulator + if (LlidarBetaMol532gr) allocate(x%grLidar532_beta_mol(Npoints,Nlevels)) + if (Latb532gr) allocate(x%grLidar532_beta_tot(Npoints,Ncolumns,Nlevels)) + if (LcfadLidarsr532gr) then + allocate(x%grLidar532_srbval(SR_BINS+1)) + allocate(x%grLidar532_cfad_sr(Npoints,SR_BINS,Nlvgrid)) + endif + if (LclgrLidar532) allocate(x%grLidar532_lidarcld(Npoints,Nlvgrid)) + if (LclhgrLidar532 .or. LclmgrLidar532 .or. LcllgrLidar532 .or. LcltgrLidar532) then + allocate(x%grLidar532_cldlayer(Npoints,LIDAR_NCAT)) + endif + + ! ATLID simulator + if (LlidarBetaMol355) allocate(x%atlid_beta_mol(Npoints,Nlevels)) + if (Latb355) allocate(x%atlid_beta_tot(Npoints,Ncolumns,Nlevels)) + if (LcfadLidarsr355) then + allocate(x%atlid_srbval(SR_BINS+1)) + allocate(x%atlid_cfad_sr(Npoints,SR_BINS,Nlvgrid)) + endif + if (Lclatlid) allocate(x%atlid_lidarcld(Npoints,Nlvgrid)) + if (Lclhatlid .or. Lclmatlid .or. Lcllatlid .or. Lcltatlid) then + allocate(x%atlid_cldlayer(Npoints,LIDAR_NCAT)) + endif + + ! PARASOL + if (Lparasolrefl) then + allocate(x%parasolPix_refl(Npoints,Ncolumns,PARASOL_NREFL)) + allocate(x%parasolGrid_refl(Npoints,PARASOL_NREFL)) + endif + + ! Cloudsat simulator + if (Ldbze94) allocate(x%cloudsat_Ze_tot(Npoints,Ncolumns,Nlevels)) + if (LcfadDbze94) allocate(x%cloudsat_cfad_ze(Npoints,cloudsat_DBZE_BINS,Nlvgrid)) + if (Lptradarflag0 .or. Lptradarflag1 .or. Lptradarflag2 .or. Lptradarflag3 .or. & + Lptradarflag4 .or. Lptradarflag5 .or. Lptradarflag6 .or. Lptradarflag7 .or. & + Lptradarflag8 .or. Lptradarflag9) then + allocate(x%cloudsat_precip_cover(Npoints,cloudsat_DBZE_BINS)) + endif + if (Lradarpia) allocate(x%cloudsat_pia(Npoints)) + + ! Combined CALIPSO/CLOUDSAT fields + if (Lclcalipso2) allocate(x%lidar_only_freq_cloud(Npoints,Nlvgrid)) + if (Lcltlidarradar) allocate(x%radar_lidar_tcc(Npoints)) + if (Lcloudsat_tcc) allocate(x%cloudsat_tcc(Npoints)) + if (Lcloudsat_tcc2) allocate(x%cloudsat_tcc2(Npoints)) + + ! RTTOV + if (Ltbrttov) allocate(x%rttov_tbs(Npoints,Nchan)) + + ! Joint MODIS/CloudSat Statistics + !if (Lwr_occfreq) allocate(x%wr_occfreq_ntotal(Npoints,WR_NREGIME)) + !if (Lcfodd) allocate(x%cfodd_ntotal(Npoints,CFODD_NDBZE,CFODD_NICOD,CFODD_NCLASS)) + + end subroutine construct_cosp_outputs + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! SUBROUTINE destroy_cospIN + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + subroutine destroy_cospIN(y) + type(cosp_optical_inputs),intent(inout) :: y + if (allocated(y%tau_067)) deallocate(y%tau_067) + if (allocated(y%emiss_11)) deallocate(y%emiss_11) + if (allocated(y%frac_out)) deallocate(y%frac_out) + if (allocated(y%beta_mol_calipso)) deallocate(y%beta_mol_calipso) + if (allocated(y%tau_mol_calipso)) deallocate(y%tau_mol_calipso) + if (allocated(y%betatot_calipso)) deallocate(y%betatot_calipso) + if (allocated(y%betatot_ice_calipso)) deallocate(y%betatot_ice_calipso) + if (allocated(y%betatot_liq_calipso)) deallocate(y%betatot_liq_calipso) + if (allocated(y%tautot_calipso)) deallocate(y%tautot_calipso) + if (allocated(y%tautot_ice_calipso)) deallocate(y%tautot_ice_calipso) + if (allocated(y%tautot_liq_calipso)) deallocate(y%tautot_liq_calipso) + if (allocated(y%tautot_S_liq)) deallocate(y%tautot_S_liq) + if (allocated(y%tautot_S_ice)) deallocate(y%tautot_S_ice) + if (allocated(y%z_vol_cloudsat)) deallocate(y%z_vol_cloudsat) + if (allocated(y%kr_vol_cloudsat)) deallocate(y%kr_vol_cloudsat) + if (allocated(y%g_vol_cloudsat)) deallocate(y%g_vol_cloudsat) + if (allocated(y%asym)) deallocate(y%asym) + if (allocated(y%ss_alb)) deallocate(y%ss_alb) + if (allocated(y%fracLiq)) deallocate(y%fracLiq) + if (allocated(y%beta_mol_grLidar532)) deallocate(y%beta_mol_grLidar532) + if (allocated(y%betatot_grLidar532)) deallocate(y%betatot_grLidar532) + if (allocated(y%tau_mol_grLidar532)) deallocate(y%tau_mol_grLidar532) + if (allocated(y%tautot_grLidar532)) deallocate(y%tautot_grLidar532) + if (allocated(y%beta_mol_atlid)) deallocate(y%beta_mol_atlid) + if (allocated(y%betatot_atlid)) deallocate(y%betatot_atlid) + if (allocated(y%tau_mol_atlid)) deallocate(y%tau_mol_atlid) + if (allocated(y%tautot_atlid)) deallocate(y%tautot_atlid) + if (allocated(y%fracPrecipIce)) deallocate(y%fracPrecipIce) + end subroutine destroy_cospIN + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! SUBROUTINE destroy_cospstateIN + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + subroutine destroy_cospstateIN(y) + type(cosp_column_inputs),intent(inout) :: y + + if (allocated(y%sunlit)) deallocate(y%sunlit) + if (allocated(y%skt)) deallocate(y%skt) + if (allocated(y%land)) deallocate(y%land) + if (allocated(y%at)) deallocate(y%at) + if (allocated(y%pfull)) deallocate(y%pfull) + if (allocated(y%phalf)) deallocate(y%phalf) + if (allocated(y%qv)) deallocate(y%qv) + if (allocated(y%o3)) deallocate(y%o3) + if (allocated(y%hgt_matrix)) deallocate(y%hgt_matrix) + if (allocated(y%u_sfc)) deallocate(y%u_sfc) + if (allocated(y%v_sfc)) deallocate(y%v_sfc) + if (allocated(y%lat)) deallocate(y%lat) + if (allocated(y%lon)) deallocate(y%lon) + if (allocated(y%emis_sfc)) deallocate(y%emis_sfc) + if (allocated(y%cloudIce)) deallocate(y%cloudIce) + if (allocated(y%cloudLiq)) deallocate(y%cloudLiq) + if (allocated(y%seaice)) deallocate(y%seaice) + if (allocated(y%fl_rain)) deallocate(y%fl_rain) + if (allocated(y%fl_snow)) deallocate(y%fl_snow) + if (allocated(y%tca)) deallocate(y%tca) + if (allocated(y%hgt_matrix_half)) deallocate(y%hgt_matrix_half) + if (allocated(y%surfelev)) deallocate(y%surfelev) + + end subroutine destroy_cospstateIN + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! SUBROUTINE destroy_cosp_outputs + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + subroutine destroy_cosp_outputs(y) + type(cosp_outputs),intent(inout) :: y + + ! Deallocate and nullify + if (associated(y%calipso_beta_mol)) then + deallocate(y%calipso_beta_mol) + nullify(y%calipso_beta_mol) + endif + if (associated(y%calipso_temp_tot)) then + deallocate(y%calipso_temp_tot) + nullify(y%calipso_temp_tot) + endif + if (associated(y%calipso_betaperp_tot)) then + deallocate(y%calipso_betaperp_tot) + nullify(y%calipso_betaperp_tot) + endif + if (associated(y%calipso_beta_tot)) then + deallocate(y%calipso_beta_tot) + nullify(y%calipso_beta_tot) + endif + if (associated(y%calipso_tau_tot)) then + deallocate(y%calipso_tau_tot) + nullify(y%calipso_tau_tot) + endif + if (associated(y%calipso_lidarcldphase)) then + deallocate(y%calipso_lidarcldphase) + nullify(y%calipso_lidarcldphase) + endif + if (associated(y%calipso_lidarcldtype)) then + deallocate(y%calipso_lidarcldtype) + nullify(y%calipso_lidarcldtype) + endif + if (associated(y%calipso_cldlayerphase)) then + deallocate(y%calipso_cldlayerphase) + nullify(y%calipso_cldlayerphase) + endif + if (associated(y%calipso_lidarcldtmp)) then + deallocate(y%calipso_lidarcldtmp) + nullify(y%calipso_lidarcldtmp) + endif + if (associated(y%calipso_cldlayer)) then + deallocate(y%calipso_cldlayer) + nullify(y%calipso_cldlayer) + endif + if (associated(y%calipso_cldtype)) then + deallocate(y%calipso_cldtype) + nullify(y%calipso_cldtype) + endif + if (associated(y%calipso_cldtypetemp)) then + deallocate(y%calipso_cldtypetemp) + nullify(y%calipso_cldtypetemp) + endif + if (associated(y%calipso_cldtypemeanz)) then + deallocate(y%calipso_cldtypemeanz) + nullify(y%calipso_cldtypemeanz) + endif + if (associated(y%calipso_cldtypemeanzse)) then + deallocate(y%calipso_cldtypemeanzse) + nullify(y%calipso_cldtypemeanzse) + endif + if (associated(y%calipso_cldthinemis)) then + deallocate(y%calipso_cldthinemis) + nullify(y%calipso_cldthinemis) + endif + if (associated(y%calipso_lidarcld)) then + deallocate(y%calipso_lidarcld) + nullify(y%calipso_lidarcld) + endif + if (associated(y%calipso_srbval)) then + deallocate(y%calipso_srbval) + nullify(y%calipso_srbval) + endif + if (associated(y%calipso_cfad_sr)) then + deallocate(y%calipso_cfad_sr) + nullify(y%calipso_cfad_sr) + endif + if (associated(y%grLidar532_beta_mol)) then + deallocate(y%grLidar532_beta_mol) + nullify(y%grLidar532_beta_mol) + endif + if (associated(y%grLidar532_beta_tot)) then + deallocate(y%grLidar532_beta_tot) + nullify(y%grLidar532_beta_tot) + endif + if (associated(y%grLidar532_cldlayer)) then + deallocate(y%grLidar532_cldlayer) + nullify(y%grLidar532_cldlayer) + endif + if (associated(y%grLidar532_lidarcld)) then + deallocate(y%grLidar532_lidarcld) + nullify(y%grLidar532_lidarcld) + endif + if (associated(y%grLidar532_cfad_sr)) then + deallocate(y%grLidar532_cfad_sr) + nullify(y%grLidar532_cfad_sr) + endif + if (associated(y%grLidar532_srbval)) then + deallocate(y%grLidar532_srbval) + nullify(y%grLidar532_srbval) + endif + if (associated(y%atlid_beta_mol)) then + deallocate(y%atlid_beta_mol) + nullify(y%atlid_beta_mol) + endif + if (associated(y%atlid_beta_tot)) then + deallocate(y%atlid_beta_tot) + nullify(y%atlid_beta_tot) + endif + if (associated(y%atlid_cldlayer)) then + deallocate(y%atlid_cldlayer) + nullify(y%atlid_cldlayer) + endif + if (associated(y%atlid_lidarcld)) then + deallocate(y%atlid_lidarcld) + nullify(y%atlid_lidarcld) + endif + if (associated(y%atlid_cfad_sr)) then + deallocate(y%atlid_cfad_sr) + nullify(y%atlid_cfad_sr) + endif + if (associated(y%atlid_srbval)) then + deallocate(y%atlid_srbval) + nullify(y%atlid_srbval) + endif + if (associated(y%parasolPix_refl)) then + deallocate(y%parasolPix_refl) + nullify(y%parasolPix_refl) + endif + if (associated(y%parasolGrid_refl)) then + deallocate(y%parasolGrid_refl) + nullify(y%parasolGrid_refl) + endif + if (associated(y%cloudsat_Ze_tot)) then + deallocate(y%cloudsat_Ze_tot) + nullify(y%cloudsat_Ze_tot) + endif + if (associated(y%cloudsat_cfad_ze)) then + deallocate(y%cloudsat_cfad_ze) + nullify(y%cloudsat_cfad_ze) + endif + if (associated(y%cloudsat_precip_cover)) then + deallocate(y%cloudsat_precip_cover) + nullify(y%cloudsat_precip_cover) + endif + if (associated(y%cloudsat_pia)) then + deallocate(y%cloudsat_pia) + nullify(y%cloudsat_pia) + endif + if (associated(y%cloudsat_tcc)) then + deallocate(y%cloudsat_tcc) + nullify(y%cloudsat_tcc) + endif + if (associated(y%cloudsat_tcc2)) then + deallocate(y%cloudsat_tcc2) + nullify(y%cloudsat_tcc2) + endif + if (associated(y%radar_lidar_tcc)) then + deallocate(y%radar_lidar_tcc) + nullify(y%radar_lidar_tcc) + endif + if (associated(y%cloudsat_tcc)) then + deallocate(y%cloudsat_tcc) + nullify(y%cloudsat_tcc) + endif + if (associated(y%cloudsat_tcc2)) then + deallocate(y%cloudsat_tcc2) + nullify(y%cloudsat_tcc2) + endif + if (associated(y%lidar_only_freq_cloud)) then + deallocate(y%lidar_only_freq_cloud) + nullify(y%lidar_only_freq_cloud) + endif + if (associated(y%isccp_totalcldarea)) then + deallocate(y%isccp_totalcldarea) + nullify(y%isccp_totalcldarea) + endif + if (associated(y%isccp_meantb)) then + deallocate(y%isccp_meantb) + nullify(y%isccp_meantb) + endif + if (associated(y%isccp_meantbclr)) then + deallocate(y%isccp_meantbclr) + nullify(y%isccp_meantbclr) + endif + if (associated(y%isccp_meanptop)) then + deallocate(y%isccp_meanptop) + nullify(y%isccp_meanptop) + endif + if (associated(y%isccp_meantaucld)) then + deallocate(y%isccp_meantaucld) + nullify(y%isccp_meantaucld) + endif + if (associated(y%isccp_meanalbedocld)) then + deallocate(y%isccp_meanalbedocld) + nullify(y%isccp_meanalbedocld) + endif + if (associated(y%isccp_boxtau)) then + deallocate(y%isccp_boxtau) + nullify(y%isccp_boxtau) + endif + if (associated(y%isccp_boxptop)) then + deallocate(y%isccp_boxptop) + nullify(y%isccp_boxptop) + endif + if (associated(y%isccp_fq)) then + deallocate(y%isccp_fq) + nullify(y%isccp_fq) + endif + if (associated(y%misr_fq)) then + deallocate(y%misr_fq) + nullify(y%misr_fq) + endif + if (associated(y%misr_dist_model_layertops)) then + deallocate(y%misr_dist_model_layertops) + nullify(y%misr_dist_model_layertops) + endif + if (associated(y%misr_meanztop)) then + deallocate(y%misr_meanztop) + nullify(y%misr_meanztop) + endif + if (associated(y%misr_cldarea)) then + deallocate(y%misr_cldarea) + nullify(y%misr_cldarea) + endif + if (associated(y%rttov_tbs)) then + deallocate(y%rttov_tbs) + nullify(y%rttov_tbs) + endif + if (associated(y%modis_Cloud_Fraction_Total_Mean)) then + deallocate(y%modis_Cloud_Fraction_Total_Mean) + nullify(y%modis_Cloud_Fraction_Total_Mean) + endif + if (associated(y%modis_Cloud_Fraction_Ice_Mean)) then + deallocate(y%modis_Cloud_Fraction_Ice_Mean) + nullify(y%modis_Cloud_Fraction_Ice_Mean) + endif + if (associated(y%modis_Cloud_Fraction_Water_Mean)) then + deallocate(y%modis_Cloud_Fraction_Water_Mean) + nullify(y%modis_Cloud_Fraction_Water_Mean) + endif + if (associated(y%modis_Cloud_Fraction_High_Mean)) then + deallocate(y%modis_Cloud_Fraction_High_Mean) + nullify(y%modis_Cloud_Fraction_High_Mean) + endif + if (associated(y%modis_Cloud_Fraction_Mid_Mean)) then + deallocate(y%modis_Cloud_Fraction_Mid_Mean) + nullify(y%modis_Cloud_Fraction_Mid_Mean) + endif + if (associated(y%modis_Cloud_Fraction_Low_Mean)) then + deallocate(y%modis_Cloud_Fraction_Low_Mean) + nullify(y%modis_Cloud_Fraction_Low_Mean) + endif + if (associated(y%modis_Optical_Thickness_Total_Mean)) then + deallocate(y%modis_Optical_Thickness_Total_Mean) + nullify(y%modis_Optical_Thickness_Total_Mean) + endif + if (associated(y%modis_Optical_Thickness_Water_Mean)) then + deallocate(y%modis_Optical_Thickness_Water_Mean) + nullify(y%modis_Optical_Thickness_Water_Mean) + endif + if (associated(y%modis_Optical_Thickness_Ice_Mean)) then + deallocate(y%modis_Optical_Thickness_Ice_Mean) + nullify(y%modis_Optical_Thickness_Ice_Mean) + endif + if (associated(y%modis_Optical_Thickness_Total_LogMean)) then + deallocate(y%modis_Optical_Thickness_Total_LogMean) + nullify(y%modis_Optical_Thickness_Total_LogMean) + endif + if (associated(y%modis_Optical_Thickness_Water_LogMean)) then + deallocate(y%modis_Optical_Thickness_Water_LogMean) + nullify(y%modis_Optical_Thickness_Water_LogMean) + endif + if (associated(y%modis_Optical_Thickness_Ice_LogMean)) then + deallocate(y%modis_Optical_Thickness_Ice_LogMean) + nullify(y%modis_Optical_Thickness_Ice_LogMean) + endif + if (associated(y%modis_Cloud_Particle_Size_Water_Mean)) then + deallocate(y%modis_Cloud_Particle_Size_Water_Mean) + nullify(y%modis_Cloud_Particle_Size_Water_Mean) + endif + if (associated(y%modis_Cloud_Particle_Size_Ice_Mean)) then + deallocate(y%modis_Cloud_Particle_Size_Ice_Mean) + nullify(y%modis_Cloud_Particle_Size_Ice_Mean) + endif + if (associated(y%modis_Cloud_Top_Pressure_Total_Mean)) then + deallocate(y%modis_Cloud_Top_Pressure_Total_Mean) + nullify(y%modis_Cloud_Top_Pressure_Total_Mean) + endif + if (associated(y%modis_Liquid_Water_Path_Mean)) then + deallocate(y%modis_Liquid_Water_Path_Mean) + nullify(y%modis_Liquid_Water_Path_Mean) + endif + if (associated(y%modis_Ice_Water_Path_Mean)) then + deallocate(y%modis_Ice_Water_Path_Mean) + nullify(y%modis_Ice_Water_Path_Mean) + endif + if (associated(y%modis_Optical_Thickness_vs_Cloud_Top_Pressure)) then + deallocate(y%modis_Optical_Thickness_vs_Cloud_Top_Pressure) + nullify(y%modis_Optical_Thickness_vs_Cloud_Top_Pressure) + endif + if (associated(y%modis_Optical_thickness_vs_ReffLIQ)) then + deallocate(y%modis_Optical_thickness_vs_ReffLIQ) + nullify(y%modis_Optical_thickness_vs_ReffLIQ) + endif + if (associated(y%modis_Optical_thickness_vs_ReffICE)) then + deallocate(y%modis_Optical_thickness_vs_ReffICE) + nullify(y%modis_Optical_thickness_vs_ReffICE) + endif + !if (associated(y%cfodd_ntotal)) then + ! deallocate(y%cfodd_ntotal) + ! nullify(y%cfodd_ntotal) + !endif + !if (associated(y%wr_occfreq_ntotal)) then + ! deallocate(y%wr_occfreq_ntotal) + ! nullify(y%wr_occfreq_ntotal) + !endif + + end subroutine destroy_cosp_outputs + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! SUBROUTINE subsample_and_optics + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + subroutine subsample_and_optics(nPoints, nLevels, nColumns, nHydro, overlap, use_vgrid, & + lidar_ice_type, sd, tca, cca, fl_lsrainIN, fl_lssnowIN, & + fl_lsgrplIN, fl_ccrainIN, fl_ccsnowIN, mr_lsliq, mr_lsice, mr_ccliq, mr_ccice, & + reffIN, dtau_c, dtau_s, dem_c, dem_s, cospstateIN, cospIN) + ! Inputs + integer,intent(in) :: nPoints, nLevels, nColumns, nHydro, overlap, lidar_ice_type + real(wp),intent(in),dimension(nPoints,nLevels) :: tca,cca,mr_lsliq,mr_lsice,mr_ccliq, & + mr_ccice,dtau_c,dtau_s,dem_c,dem_s,fl_lsrainIN,fl_lssnowIN,fl_lsgrplIN,fl_ccrainIN,& + fl_ccsnowIN + real(wp),intent(in),dimension(nPoints,nLevels,nHydro) :: reffIN + logical,intent(in) :: use_vgrid ! .false.: outputs on model levels + ! .true.: outputs on evenly-spaced vertical levels. + type(size_distribution),intent(inout) :: sd + + ! Outputs + type(cosp_optical_inputs),intent(inout) :: cospIN + type(cosp_column_inputs),intent(inout) :: cospstateIN + + ! Local variables + type(rng_state),allocatable,dimension(:) :: rngs ! Seeds for random number generator + integer,dimension(:),allocatable :: seed + integer,dimension(:),allocatable :: cloudsat_preclvl_index + integer :: i,j,k + real(wp) :: zstep + real(wp),dimension(:,:), allocatable :: & + ls_p_rate, cv_p_rate, frac_ls, frac_cv, prec_ls, prec_cv,g_vol + real(wp),dimension(:,:,:), allocatable :: & + frac_prec, MODIS_cloudWater, MODIS_cloudIce, fracPrecipIce, fracPrecipIce_statGrid,& + MODIS_watersize,MODIS_iceSize, MODIS_opticalThicknessLiq,MODIS_opticalThicknessIce + real(wp),dimension(:,:,:,:),allocatable :: & + mr_hydro, Reff, Np + real(wp),dimension(nPoints,nLevels) :: & + column_frac_out, column_prec_out, fl_lsrain, fl_lssnow, fl_lsgrpl, fl_ccrain, fl_ccsnow + real(wp),dimension(nPoints,nColumns,Nlvgrid) :: tempOut + logical :: cmpGases=.true. + + + if (Ncolumns .gt. 1) then + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! Generate subcolumns for clouds (SCOPS) and precipitation type (PREC_SCOPS) + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! RNG used for subcolumn generation + allocate(rngs(nPoints),seed(nPoints)) + seed(:)=0 + seed = int(cospstateIN%phalf(:,Nlevels+1)) ! In case of NPoints=1 + ! *NOTE* Chunking will change the seed + if (NPoints .gt. 1) seed=int((cospstateIN%phalf(:,Nlevels+1)-minval(cospstateIN%phalf(:,Nlevels+1)))/ & + (maxval(cospstateIN%phalf(:,Nlevels+1))-minval(cospstateIN%phalf(:,Nlevels+1)))*100000) + 1 + call init_rng(rngs, seed) + + ! Call scops + call scops(NPoints,Nlevels,Ncolumns,rngs,tca,cca,overlap,cospIN%frac_out,0) + deallocate(seed,rngs) + + ! Sum up precipitation rates + allocate(ls_p_rate(nPoints,nLevels),cv_p_rate(nPoints,Nlevels)) + ls_p_rate(:,1:nLevels) = 0 ! mixing_ratio(rain) + mixing_ratio(snow) + mixing_ratio (groupel) + cv_p_rate(:,1:nLevels) = 0 ! mixing_ratio(rain) + mixing_ratio(snow) + + ! Call PREC_SCOPS + allocate(frac_prec(nPoints,nColumns,nLevels)) + call prec_scops(nPoints,nLevels,nColumns,ls_p_rate,cv_p_rate,cospIN%frac_out,frac_prec) + deallocate(ls_p_rate,cv_p_rate) + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! Compute fraction in each gridbox for precipitation and cloud type. + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! Allocate + allocate(frac_ls(nPoints,nLevels),prec_ls(nPoints,nLevels), & + frac_cv(nPoints,nLevels),prec_cv(nPoints,nLevels)) + + ! Initialize + frac_ls(1:nPoints,1:nLevels) = 0._wp + prec_ls(1:nPoints,1:nLevels) = 0._wp + frac_cv(1:nPoints,1:nLevels) = 0._wp + prec_cv(1:nPoints,1:nLevels) = 0._wp + do j=1,nPoints + do k=1,nLevels + do i=1,nColumns + if (cospIN%frac_out(j,i,k) .eq. 1) frac_ls(j,k) = frac_ls(j,k)+1._wp + if (cospIN%frac_out(j,i,k) .eq. 2) frac_cv(j,k) = frac_cv(j,k)+1._wp + if (frac_prec(j,i,k) .eq. 1) prec_ls(j,k) = prec_ls(j,k)+1._wp + if (frac_prec(j,i,k) .eq. 2) prec_cv(j,k) = prec_cv(j,k)+1._wp + if (frac_prec(j,i,k) .eq. 3) prec_cv(j,k) = prec_cv(j,k)+1._wp + if (frac_prec(j,i,k) .eq. 3) prec_ls(j,k) = prec_ls(j,k)+1._wp + enddo + frac_ls(j,k)=frac_ls(j,k)/nColumns + frac_cv(j,k)=frac_cv(j,k)/nColumns + prec_ls(j,k)=prec_ls(j,k)/nColumns + prec_cv(j,k)=prec_cv(j,k)/nColumns + enddo + enddo + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! Assign gridmean mixing-ratios (mr_XXXXX), effective radius (ReffIN) and number + ! concentration (not defined) to appropriate sub-column. Here we are using scops. + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + allocate(mr_hydro(nPoints,nColumns,nLevels,nHydro), & + Reff(nPoints,nColumns,nLevels,nHydro), & + Np(nPoints,nColumns,nLevels,nHydro)) + + ! Initialize + mr_hydro(:,:,:,:) = 0._wp + Reff(:,:,:,:) = 0._wp + Np(:,:,:,:) = 0._wp + do k=1,nColumns + ! Subcolumn cloud fraction + column_frac_out = cospIN%frac_out(:,k,:) + + ! LS clouds + where (column_frac_out == I_LSC) + mr_hydro(:,k,:,I_LSCLIQ) = mr_lsliq + mr_hydro(:,k,:,I_LSCICE) = mr_lsice + Reff(:,k,:,I_LSCLIQ) = ReffIN(:,:,I_LSCLIQ) + Reff(:,k,:,I_LSCICE) = ReffIN(:,:,I_LSCICE) + ! CONV clouds + elsewhere (column_frac_out == I_CVC) + mr_hydro(:,k,:,I_CVCLIQ) = mr_ccliq + mr_hydro(:,k,:,I_CVCICE) = mr_ccice + Reff(:,k,:,I_CVCLIQ) = ReffIN(:,:,I_CVCLIQ) + Reff(:,k,:,I_CVCICE) = ReffIN(:,:,I_CVCICE) + end where + + ! Subcolumn precipitation + column_prec_out = frac_prec(:,k,:) + + ! LS Precipitation + where ((column_prec_out == 1) .or. (column_prec_out == 3) ) + Reff(:,k,:,I_LSRAIN) = ReffIN(:,:,I_LSRAIN) + Reff(:,k,:,I_LSSNOW) = ReffIN(:,:,I_LSSNOW) + Reff(:,k,:,I_LSGRPL) = ReffIN(:,:,I_LSGRPL) + ! CONV precipitation + elsewhere ((column_prec_out == 2) .or. (column_prec_out == 3)) + Reff(:,k,:,I_CVRAIN) = ReffIN(:,:,I_CVRAIN) + Reff(:,k,:,I_CVSNOW) = ReffIN(:,:,I_CVSNOW) + end where + enddo + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! Convert the subcolumn mixing ratio and precipitation fluxes from gridbox mean + ! values to fraction-based values. + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! Initialize + fl_lsrain(:,:) = 0._wp + fl_lssnow(:,:) = 0._wp + fl_lsgrpl(:,:) = 0._wp + fl_ccrain(:,:) = 0._wp + fl_ccsnow(:,:) = 0._wp + do k=1,nLevels + do j=1,nPoints + ! In-cloud mixing ratios. + if (frac_ls(j,k) .ne. 0.) then + mr_hydro(j,:,k,I_LSCLIQ) = mr_hydro(j,:,k,I_LSCLIQ)/frac_ls(j,k) + mr_hydro(j,:,k,I_LSCICE) = mr_hydro(j,:,k,I_LSCICE)/frac_ls(j,k) + endif + if (frac_cv(j,k) .ne. 0.) then + mr_hydro(j,:,k,I_CVCLIQ) = mr_hydro(j,:,k,I_CVCLIQ)/frac_cv(j,k) + mr_hydro(j,:,k,I_CVCICE) = mr_hydro(j,:,k,I_CVCICE)/frac_cv(j,k) + endif + ! Precipitation + if (prec_ls(j,k) .ne. 0.) then + mr_hydro(j,:,k,I_LSRAIN) = mr_hydro(j,:,k,I_LSRAIN)/prec_ls(j,k) + mr_hydro(j,:,k,I_LSSNOW) = mr_hydro(j,:,k,I_LSSNOW)/prec_ls(j,k) + mr_hydro(j,:,k,I_LSGRPL) = mr_hydro(j,:,k,I_LSGRPL)/prec_ls(j,k) + endif + if (prec_cv(j,k) .ne. 0.) then + mr_hydro(j,:,k,I_CVRAIN) = mr_hydro(j,:,k,I_CVRAIN)/prec_cv(j,k) + mr_hydro(j,:,k,I_CVSNOW) = mr_hydro(j,:,k,I_CVSNOW)/prec_cv(j,k) + endif + enddo + enddo + deallocate(frac_ls,prec_ls,frac_cv,prec_cv) + + else + cospIN%frac_out(:,:,:) = 1 + allocate(mr_hydro(nPoints,1,nLevels,nHydro),Reff(nPoints,1,nLevels,nHydro), & + Np(nPoints,1,nLevels,nHydro)) + mr_hydro(:,1,:,I_LSCLIQ) = mr_lsliq + mr_hydro(:,1,:,I_LSCICE) = mr_lsice + mr_hydro(:,1,:,I_CVCLIQ) = mr_ccliq + mr_hydro(:,1,:,I_CVCICE) = mr_ccice + Reff(:,1,:,:) = ReffIN + endif + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! 11 micron emissivity + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + if (Lisccp) then + call cosp_simulator_optics(nPoints,nColumns,nLevels,cospIN%frac_out,dem_c,dem_s, & + cospIN%emiss_11) + endif + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! 0.67 micron optical depth + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + if (Lisccp .or. Lmisr .or. Lmodis) then + call cosp_simulator_optics(nPoints,nColumns,nLevels,cospIN%frac_out,dtau_c,dtau_s, & + cospIN%tau_067) + endif + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! LIDAR Polarized optics + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + if (Lcalipso) then + call lidar_optics(nPoints, nColumns, nLevels, 4, lidar_ice_type, 532, .false., & + mr_hydro(:,:,:,I_LSCLIQ), mr_hydro(:,:,:,I_LSCICE), mr_hydro(:,:,:,I_CVCLIQ), & + mr_hydro(:,:,:,I_CVCICE), ReffIN(:,:,I_LSCLIQ), ReffIN(:,:,I_LSCICE), & + ReffIN(:,:,I_CVCLIQ), ReffIN(:,:,I_CVCICE), cospstateIN%pfull, & + cospstateIN%phalf, cospstateIN%at, cospIN%beta_mol_calipso, & + cospIN%betatot_calipso, cospIN%tau_mol_calipso, cospIN%tautot_calipso, & + cospIN%tautot_S_liq, cospIN%tautot_S_ice, cospIN%betatot_ice_calipso, & + cospIN%betatot_liq_calipso, cospIN%tautot_ice_calipso, cospIN%tautot_liq_calipso) + endif + + if (LgrLidar532) then + call lidar_optics(nPoints, nColumns, nLevels, 4, lidar_ice_type, 532, .true., & + mr_hydro(:,:,:,I_LSCLIQ), mr_hydro(:,:,:,I_LSCICE), mr_hydro(:,:,:,I_CVCLIQ), & + mr_hydro(:,:,:,I_CVCICE), ReffIN(:,:,I_LSCLIQ), ReffIN(:,:,I_LSCICE), & + ReffIN(:,:,I_CVCLIQ), ReffIN(:,:,I_CVCICE), cospstateIN%pfull, & + cospstateIN%phalf, cospstateIN%at, cospIN%beta_mol_grLidar532, & + cospIN%betatot_grLidar532, cospIN%tau_mol_grLidar532, cospIN%tautot_grLidar532) + endif + + if (Latlid) then + call lidar_optics(nPoints, nColumns, nLevels, 4, lidar_ice_type, 355, .false., & + mr_hydro(:,:,:,I_LSCLIQ), mr_hydro(:,:,:,I_LSCICE), mr_hydro(:,:,:,I_CVCLIQ), & + mr_hydro(:,:,:,I_CVCICE), ReffIN(:,:,I_LSCLIQ), ReffIN(:,:,I_LSCICE), & + ReffIN(:,:,I_CVCLIQ), ReffIN(:,:,I_CVCICE), cospstateIN%pfull, & + cospstateIN%phalf, cospstateIN%at, cospIN%beta_mol_atlid, cospIN%betatot_atlid,& + cospIN%tau_mol_atlid, cospIN%tautot_atlid) + endif + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! CLOUDSAT RADAR OPTICS + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + if (lcloudsat) then + + ! Compute gaseous absorption (assume identical for each subcolun) + allocate(g_vol(nPoints,nLevels)) + g_vol(:,:)=0._wp + do i=1,nPoints + do j=1,nLevels + if (rcfg_cloudsat%use_gas_abs == 1 .or. (rcfg_cloudsat%use_gas_abs == 2 .and. j .eq. 1)) then + g_vol(i,j) = gases(cospstateIN%pfull(i,j), cospstateIN%at(i,j),cospstateIN%qv(i,j),rcfg_cloudsat%freq) + endif + cospIN%g_vol_cloudsat(i,:,j)=g_vol(i,j) + end do + end do + + ! Loop over all subcolumns + allocate(fracPrecipIce(nPoints,nColumns,nLevels)) + fracPrecipIce(:,:,:) = 0._wp + do k=1,nColumns + call quickbeam_optics(sd, rcfg_cloudsat, nPoints, nLevels, R_UNDEF, & + mr_hydro(:,k,:,1:nHydro)*1000._wp, Reff(:,k,:,1:nHydro)*1.e6_wp,& + Np(:,k,:,1:nHydro), cospstateIN%pfull, cospstateIN%at, & + cospstateIN%qv, cospIN%z_vol_cloudsat(1:nPoints,k,:), & + cospIN%kr_vol_cloudsat(1:nPoints,k,:)) + + ! At each model level, what fraction of the precipitation is frozen? + where(mr_hydro(:,k,:,I_LSRAIN) .gt. 0 .or. mr_hydro(:,k,:,I_LSSNOW) .gt. 0 .or. & + mr_hydro(:,k,:,I_CVRAIN) .gt. 0 .or. mr_hydro(:,k,:,I_CVSNOW) .gt. 0 .or. & + mr_hydro(:,k,:,I_LSGRPL) .gt. 0) + fracPrecipIce(:,k,:) = (mr_hydro(:,k,:,I_LSSNOW) + mr_hydro(:,k,:,I_CVSNOW) + & + mr_hydro(:,k,:,I_LSGRPL)) / & + (mr_hydro(:,k,:,I_LSSNOW) + mr_hydro(:,k,:,I_CVSNOW) + mr_hydro(:,k,:,I_LSGRPL) + & + mr_hydro(:,k,:,I_LSRAIN) + mr_hydro(:,k,:,I_CVRAIN)) + elsewhere + fracPrecipIce(:,k,:) = 0._wp + endwhere + enddo + + ! Regrid frozen fraction to Cloudsat/Calipso statistical grid + if (use_vgrid) then + allocate(fracPrecipIce_statGrid(nPoints,nColumns,Nlvgrid)) + fracPrecipIce_statGrid(:,:,:) = 0._wp + call cosp_change_vertical_grid(Npoints, Ncolumns, Nlevels, cospstateIN%hgt_matrix(:,Nlevels:1:-1), & + cospstateIN%hgt_matrix_half(:,Nlevels:1:-1), fracPrecipIce(:,:,Nlevels:1:-1), Nlvgrid, & + vgrid_zl(Nlvgrid:1:-1), vgrid_zu(Nlvgrid:1:-1), fracPrecipIce_statGrid(:,:,Nlvgrid:1:-1)) + + ! Find proper layer above de surface elevation to compute precip flags in Cloudsat/Calipso statistical grid + allocate(cloudsat_preclvl_index(nPoints)) + cloudsat_preclvl_index(:) = 0._wp + ! Compute the zstep distance between two atmopsheric layers + zstep = vgrid_zl(1)-vgrid_zl(2) + ! Computing altitude index for precip flags calculation (one layer above surfelev layer) + cloudsat_preclvl_index(:) = cloudsat_preclvl - floor( cospstateIN%surfelev(:)/zstep ) + + ! For near-surface diagnostics, we only need the frozen fraction at one layer. + do i=1,nPoints + cospIN%fracPrecipIce(i,:) = fracPrecipIce_statGrid(i,:,cloudsat_preclvl_index(i)) + enddo + deallocate(cloudsat_preclvl_index) + deallocate(fracPrecipIce_statGrid) + endif + + endif + + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + ! MODIS optics + !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + if (Lmodis) then + allocate(MODIS_cloudWater(nPoints,nColumns,nLevels), & + MODIS_cloudIce(nPoints,nColumns,nLevels), & + MODIS_waterSize(nPoints,nColumns,nLevels), & + MODIS_iceSize(nPoints,nColumns,nLevels), & + MODIS_opticalThicknessLiq(nPoints,nColumns,nLevels), & + MODIS_opticalThicknessIce(nPoints,nColumns,nLevels)) + ! Cloud water + call cosp_simulator_optics(nPoints,nColumns,nLevels,cospIN%frac_out, & + mr_hydro(:,:,:,I_CVCLIQ),mr_hydro(:,:,:,I_LSCLIQ),MODIS_cloudWater) + ! Cloud ice + call cosp_simulator_optics(nPoints,nColumns,nLevels,cospIN%frac_out, & + mr_hydro(:,:,:,I_CVCICE),mr_hydro(:,:,:,I_LSCICE),MODIS_cloudIce) + ! Water droplet size + call cosp_simulator_optics(nPoints,nColumns,nLevels,cospIN%frac_out, & + Reff(:,:,:,I_CVCLIQ),Reff(:,:,:,I_LSCLIQ),MODIS_waterSize) + ! Ice crystal size + call cosp_simulator_optics(nPoints,nColumns,nLevels,cospIN%frac_out, & + Reff(:,:,:,I_CVCICE),Reff(:,:,:,I_LSCICE),MODIS_iceSize) + + ! Partition optical thickness into liquid and ice parts + call modis_optics_partition(nPoints, nLevels, nColumns, MODIS_cloudWater, & + MODIS_cloudIce, MODIS_waterSize, MODIS_iceSize, cospIN%tau_067, & + MODIS_opticalThicknessLiq, MODIS_opticalThicknessIce) + + ! Compute assymetry parameter and single scattering albedo + call modis_optics(nPoints, nLevels, nColumns, MODIS_opticalThicknessLiq, & + MODIS_waterSize*1.0e6_wp, MODIS_opticalThicknessIce, & + MODIS_iceSize*1.0e6_wp, cospIN%fracLiq, cospIN%asym, cospIN%ss_alb) + + ! Deallocate memory + deallocate(MODIS_cloudWater,MODIS_cloudIce,MODIS_WaterSize,MODIS_iceSize, & + MODIS_opticalThicknessLiq,MODIS_opticalThicknessIce,mr_hydro, & + Np,Reff) + endif + end subroutine subsample_and_optics + + + +end module cosp_c2f diff --git a/components/eamxx/src/physics/cosp/cosp_functions.hpp b/components/eamxx/src/physics/cosp/cosp_functions.hpp new file mode 100644 index 000000000000..f969cdcfed1b --- /dev/null +++ b/components/eamxx/src/physics/cosp/cosp_functions.hpp @@ -0,0 +1,88 @@ +#ifndef SCREAM_COSP_FUNCTIONS_HPP +#define SCREAM_COSP_FUNCTIONS_HPP +#include "share/scream_types.hpp" +using scream::Real; +extern "C" void cosp_c2f_init(int ncol, int nsubcol, int nlay); +extern "C" void cosp_c2f_final(); +extern "C" void cosp_c2f_run(const int ncol, const int nsubcol, const int nlay, const int ntau, const int nctp, + const Real emsfc_lw, const Real* sunlit, const Real* skt, + const Real* T_mid, const Real* p_mid, const Real* p_int, const Real* qv, + const Real* cldfrac, const Real* reff_qc, const Real* reff_qi, const Real* dtau067, const Real* dtau105, + Real* isccp_cldtot, Real* isccp_ctptau); + +namespace scream { + + namespace CospFunc { + using lview_host_1d = typename ekat::KokkosTypes::template lview; + using lview_host_2d = typename ekat::KokkosTypes::template lview; + using lview_host_3d = typename ekat::KokkosTypes::template lview; + template + using view_1d = typename ekat::KokkosTypes::template view_1d; + template + using view_2d = typename ekat::KokkosTypes::template view_2d; + template + using view_3d = typename ekat::KokkosTypes::template view_3d; + + inline void initialize(int ncol, int nsubcol, int nlay) { + cosp_c2f_init(ncol, nsubcol, nlay); + }; + inline void finalize() { + cosp_c2f_final(); + }; + inline void main( + const Int ncol, const Int nsubcol, const Int nlay, const Int ntau, const Int nctp, const Real emsfc_lw, + view_1d& sunlit , view_1d& skt, + view_2d& T_mid , view_2d& p_mid , view_2d& p_int, + view_2d& qv , view_2d& cldfrac, + view_2d& reff_qc, view_2d& reff_qi, + view_2d& dtau067, view_2d& dtau105, + view_1d& isccp_cldtot , view_3d& isccp_ctptau) { + + // Make host copies and permute data as needed + lview_host_2d + T_mid_h("T_mid_h", ncol, nlay), p_mid_h("p_mid_h", ncol, nlay), p_int_h("p_int_h", ncol, nlay+1), + qv_h("qv_h", ncol, nlay), cldfrac_h("cldfrac_h", ncol, nlay), + reff_qc_h("reff_qc_h", ncol, nlay), reff_qi_h("reff_qi_h", ncol, nlay), + dtau067_h("dtau_067_h", ncol, nlay), dtau105_h("dtau105_h", ncol, nlay); + lview_host_3d isccp_ctptau_h("isccp_ctptau_h", ncol, ntau, nctp); + + // Copy to layoutLeft host views + for (int i = 0; i < ncol; i++) { + for (int j = 0; j < nlay; j++) { + T_mid_h(i,j) = T_mid(i,j); + p_mid_h(i,j) = p_mid(i,j); + qv_h(i,j) = qv(i,j); + cldfrac_h(i,j) = cldfrac(i,j); + reff_qc_h(i,j) = reff_qc(i,j); + reff_qi_h(i,j) = reff_qi(i,j); + dtau067_h(i,j) = dtau067(i,j); + dtau105_h(i,j) = dtau105(i,j); + } + } + for (int i = 0; i < ncol; i++) { + for (int j = 0; j < nlay+1; j++) { + p_int_h(i,j) = p_int(i,j); + } + } + + // Subsample here? + + // Call COSP wrapper + cosp_c2f_run(ncol, nsubcol, nlay, ntau, nctp, + emsfc_lw, sunlit.data(), skt.data(), T_mid_h.data(), p_mid_h.data(), p_int_h.data(), + qv_h.data(), + cldfrac_h.data(), reff_qc_h.data(), reff_qi_h.data(), dtau067_h.data(), dtau105_h.data(), + isccp_cldtot.data(), isccp_ctptau_h.data()); + + // Copy outputs back to layoutRight views + for (int i = 0; i < ncol; i++) { + for (int j = 0; j < ntau; j++) { + for (int k = 0; k < nctp; k++) { + isccp_ctptau(i,j,k) = isccp_ctptau_h(i,j,k); + } + } + } + } + } +} +#endif /* SCREAM_COSP_FUNCTIONS_HPP */ diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp new file mode 100644 index 000000000000..7aaa4be3b2b0 --- /dev/null +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -0,0 +1,215 @@ +#include "eamxx_cosp.hpp" +#include "cosp_functions.hpp" +#include "share/property_checks/field_within_interval_check.hpp" + +#include "ekat/ekat_assert.hpp" +#include "ekat/util/ekat_units.hpp" + +#include "share/field/field_utils.hpp" + +#include + +namespace scream +{ +// ========================================================================================= +Cosp::Cosp (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereProcess(comm, params) +{ + // Determine how often to call COSP; units can be steps or hours + m_cosp_frequency = m_params.get("cosp_frequency", 1); + m_cosp_frequency_units = m_params.get("cosp_frequency_units", "steps"); + EKAT_REQUIRE_MSG( + (m_cosp_frequency_units == "steps") || (m_cosp_frequency_units == "hours"), + "cosp_frequency_units " + m_cosp_frequency_units + " not supported" + ); + + // How many subcolumns to use for COSP + m_num_subcols = m_params.get("cosp_subcolumns", 10); +} + +// ========================================================================================= +void Cosp::set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto Q = kg/kg; + Q.set_string("kg/kg"); + auto nondim = Units::nondimensional(); + auto percent = Units::nondimensional(); + percent.set_string("%"); + auto micron = m / 1000000; + + m_grid = grids_manager->get_grid("Physics"); + const auto& grid_name = m_grid->name(); + m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces + FieldLayout scalar2d_layout { {COL}, {m_num_cols} }; + FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; + FieldLayout scalar3d_layout_int { {COL,ILEV}, {m_num_cols,m_num_levs+1} }; + FieldLayout scalar4d_layout_ctptau { {COL,ISCCPTAU,ISCCPPRS}, {m_num_cols,m_num_isccptau,m_num_isccpctp} }; + + // Set of fields used strictly as input + // Name in AD Layout Units Grid Group + add_field("surf_radiative_T", scalar2d_layout , K, grid_name); + //add_field("surfelev", scalar2d_layout , m, grid_name); + //add_field("landmask", scalar2d_layout , nondim, grid_name); + add_field("sunlit", scalar2d_layout , nondim, grid_name); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + add_field("p_int", scalar3d_layout_int, Pa, grid_name); + //add_field("height_mid", scalar3d_layout_mid, m, grid_name); + //add_field("height_int", scalar3d_layout_int, m, grid_name); + add_field("T_mid", scalar3d_layout_mid, K, grid_name); + add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers"); + add_field("qc", scalar3d_layout_mid, Q, grid_name, "tracers"); + add_field("qi", scalar3d_layout_mid, Q, grid_name, "tracers"); + add_field("cldfrac_tot_for_analysis", scalar3d_layout_mid, nondim, grid_name); + // Optical properties, should be computed in radiation interface + add_field("dtau067", scalar3d_layout_mid, nondim, grid_name); // 0.67 micron optical depth + add_field("dtau105", scalar3d_layout_mid, nondim, grid_name); // 10.5 micron optical depth + // Effective radii, should be computed in either microphysics or radiation interface + // TODO: should these be meters or microns? Was meters before, but using "m" instead + // of "micron" seemed to cause prim_model_finalize to throw error with the following: + // ABORTING WITH ERROR: Error! prim_init_model_f90 was not called yet (or prim_finalize_f90 was already called). + // P3 defines this field with micron instead of meters units, so is this a unit conversion issue? + add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name); + add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name); + // Set of fields used strictly as output + add_field("isccp_cldtot", scalar2d_layout, percent, grid_name); + add_field("isccp_ctptau", scalar4d_layout_ctptau, percent, grid_name, 1); + add_field("isccp_mask" , scalar2d_layout, nondim, grid_name); + +} + +// ========================================================================================= +void Cosp::initialize_impl (const RunType /* run_type */) +{ + // Set property checks for fields in this process + CospFunc::initialize(m_num_cols, m_num_subcols, m_num_levs); + + + // Add note to output files about processing ISCCP fields that are only valid during + // daytime. This can go away once I/O can handle masked time averages. + using stratts_t = std::map; + std::list vnames = {"isccp_cldtot", "isccp_ctptau"}; + for (const auto field_name : {"isccp_cldtot", "isccp_ctptau"}) { + auto& f = get_field_out(field_name); + auto& atts = f.get_header().get_extra_data("io: string attributes"); + atts["note"] = "Night values are zero; divide by isccp_mask to get daytime mean"; + } +} + +// ========================================================================================= +void Cosp::run_impl (const double dt) +{ + // Determine if we should update COSP this timestep + // First get frequency in steps + int cosp_freq_in_steps = 1; + if (m_cosp_frequency_units == "steps") { + cosp_freq_in_steps = m_cosp_frequency; + } else if (m_cosp_frequency_units == "hours") { + EKAT_REQUIRE_MSG((3600 % int(dt)) == 0, "cosp_frequency_units is hours but dt does not evenly divide 1 hour"); + cosp_freq_in_steps = 3600.0 * m_cosp_frequency / dt; + } else { + EKAT_ERROR_MSG("cosp_frequency_units " + m_cosp_frequency_units + " not supported"); + } + + // Make sure cosp frequency is multiple of rad frequency? + + // Compare frequency in steps with current timestep + auto ts = timestamp(); + auto update_cosp = cosp_do(cosp_freq_in_steps, ts.get_num_steps()); + + // Get fields from field manager; note that we get host views because this + // interface serves primarily as a wrapper to a c++ to f90 bridge for the COSP + // all then need to be copied to layoutLeft views to permute the indices for + // F90. + // + // Need to make sure device data is synced to host? + get_field_in("qv").sync_to_host(); + get_field_in("qc").sync_to_host(); + get_field_in("qi").sync_to_host(); + get_field_in("sunlit").sync_to_host(); + get_field_in("surf_radiative_T").sync_to_host(); + get_field_in("T_mid").sync_to_host(); + get_field_in("p_mid").sync_to_host(); + get_field_in("p_int").sync_to_host(); + get_field_in("cldfrac_tot_for_analysis").sync_to_host(); + get_field_in("eff_radius_qc").sync_to_host(); + get_field_in("eff_radius_qi").sync_to_host(); + get_field_in("dtau067").sync_to_host(); + get_field_in("dtau105").sync_to_host(); + + auto qv = get_field_in("qv").get_view(); + auto qc = get_field_in("qc").get_view(); + auto qi = get_field_in("qi").get_view(); + auto sunlit = get_field_in("sunlit").get_view(); + auto skt = get_field_in("surf_radiative_T").get_view(); + auto T_mid = get_field_in("T_mid").get_view(); + auto p_mid = get_field_in("p_mid").get_view(); + auto p_int = get_field_in("p_int").get_view(); + auto cldfrac = get_field_in("cldfrac_tot_for_analysis").get_view(); + auto reff_qc = get_field_in("eff_radius_qc").get_view(); + auto reff_qi = get_field_in("eff_radius_qi").get_view(); + auto dtau067 = get_field_in("dtau067").get_view(); + auto dtau105 = get_field_in("dtau105").get_view(); + auto isccp_cldtot = get_field_out("isccp_cldtot").get_view(); + auto isccp_ctptau = get_field_out("isccp_ctptau").get_view(); + auto isccp_mask = get_field_out("isccp_mask" ).get_view(); // Copy of sunlit flag with COSP frequency for proper averaging + + // Call COSP wrapper routines + if (update_cosp) { + Real emsfc_lw = 0.99; + Kokkos::deep_copy(isccp_mask, sunlit); + CospFunc::main( + m_num_cols, m_num_subcols, m_num_levs, m_num_isccptau, m_num_isccpctp, + emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, qv, + cldfrac, reff_qc, reff_qi, dtau067, dtau105, + isccp_cldtot, isccp_ctptau + ); + // Remask night values to ZERO since our I/O does not know how to handle masked/missing values + // in temporal averages; this is all host data, so we can just use host loops like its the 1980s + for (int i = 0; i < m_num_cols; i++) { + if (sunlit(i) == 0) { + isccp_cldtot(i) = 0; + for (int j = 0; j < m_num_isccptau; j++) { + for (int k = 0; k < m_num_isccpctp; k++) { + isccp_ctptau(i,j,k) = 0; + } + } + } + } + } else { + // If not updating COSP statistics, set these to ZERO; this essentially weights + // the ISCCP cloud properties by the sunlit mask. What will be output for time-averages + // then is the time-average mask-weighted statistics; to get true averages, we need to + // divide by the time-average of the mask. I.e., if M is the sunlit mask, and X is the ISCCP + // statistic, then + // + // avg(X) = sum(M * X) / sum(M) = (sum(M * X)/N) / (sum(M)/N) = avg(M * X) / avg(M) + // + // TODO: mask this when/if the AD ever supports masked averages + Kokkos::deep_copy(isccp_cldtot, 0.0); + Kokkos::deep_copy(isccp_ctptau, 0.0); + Kokkos::deep_copy(isccp_mask , 0.0); + } + get_field_out("isccp_cldtot").sync_to_dev(); + get_field_out("isccp_ctptau").sync_to_dev(); + get_field_out("isccp_mask" ).sync_to_dev(); +} + +// ========================================================================================= +void Cosp::finalize_impl() +{ + // Finalize COSP wrappers + CospFunc::finalize(); +} +// ========================================================================================= + +} // namespace scream diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.hpp b/components/eamxx/src/physics/cosp/eamxx_cosp.hpp new file mode 100644 index 000000000000..99f3c179a36a --- /dev/null +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.hpp @@ -0,0 +1,73 @@ +#ifndef SCREAM_COSP_HPP +#define SCREAM_COSP_HPP + +#include "share/atm_process/atmosphere_process.hpp" +#include "ekat/ekat_parameter_list.hpp" + +#include + +namespace scream +{ + +/* + * The class responsible to handle the calculation of COSP diagnostics + * The AD should store exactly ONE instance of this class stored + * in its list of subcomponents (the AD should make sure of this). +*/ + +class Cosp : public AtmosphereProcess +{ + +public: + + // Constructors + Cosp (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The type of subcomponent + AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name () const { return "cosp"; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + + inline bool cosp_do(const int icosp, const int nstep) { + // If icosp == 0, then never do cosp; + // Otherwise, we always call cosp at the first step, + // and afterwards we do cosp if the timestep is divisible + // by icosp + if (icosp == 0) { + return false; + } else { + return ( (nstep == 0) || (nstep % icosp == 0) ); + } + } + + +protected: + + // The three main overrides for the subcomponent + void initialize_impl (const RunType run_type); + void run_impl (const double dt); + void finalize_impl (); + + // cosp frequency; positive is interpreted as number of steps, negative as number of hours + int m_cosp_frequency; + ekat::CaseInsensitiveString m_cosp_frequency_units; + + // Keep track of field dimensions and the iteration count + Int m_num_cols; + Int m_num_subcols; + Int m_num_levs; + Int m_num_isccptau = 7; + Int m_num_isccpctp = 7; + Int m_num_cth = 16; + + std::shared_ptr m_grid; + +}; // class Cosp + +} // namespace scream + +#endif // SCREAM_COSP_HPP diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 80b0592111b3..a4e095bf5f2a 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -1,10 +1,13 @@ -add_library(mam eamxx_mam_microphysics.cpp) +add_library(mam + eamxx_mam_microphysics_process_interface.cpp + eamxx_mam_optics_process_interface.cpp) +target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx_proj) -target_include_directories(mam PRIVATE +target_include_directories(mam PUBLIC ${PROJECT_BINARY_DIR}/externals/haero/include ${PROJECT_BINARY_DIR}/externals/mam4xx/include ) -target_link_libraries(mam physics_share scream_share mam4xx) +target_link_libraries(mam PUBLIC physics_share scream_share mam4xx haero) #if (NOT SCREAM_LIB_ONLY) # add_subdirectory(tests) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics.hpp deleted file mode 100644 index 95e0a05713d3..000000000000 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics.hpp +++ /dev/null @@ -1,300 +0,0 @@ -#ifndef EAMXX_MAM_MICROPHYSICS_HPP -#define EAMXX_MAM_MICROPHYSICS_HPP - -#include -#include -#include - -#include -#include -#include - -#include - -#ifndef KOKKOS_ENABLE_CUDA -#define protected public -#define private public -#endif - -namespace scream -{ - -// The process responsible for handling MAM4 aerosols. The AD stores exactly ONE -// instance of this class in its list of subcomponents. -class MAMMicrophysics final : public scream::AtmosphereProcess { - using PF = scream::PhysicsFunctions; - using KT = ekat::KokkosTypes; - - // views for single- and multi-column data - using view_1d_int = typename KT::template view_1d; - using view_1d = typename KT::template view_1d; - using view_1d_const = typename KT::template view_1d; - using view_2d = typename KT::template view_2d; - using view_2d_const = typename KT::template view_2d; - - // unmanaged views (for buffer and workspace manager) - using uview_1d = Unmanaged>; - using uview_2d = Unmanaged>; - - using ColumnView = mam4::ColumnView; - using ThreadTeam = mam4::ThreadTeam; - -public: - - // Constructor - MAMMicrophysics(const ekat::Comm& comm, const ekat::ParameterList& params); - -protected: - - // -------------------------------------------------------------------------- - // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) - // -------------------------------------------------------------------------- - - // process metadata - AtmosphereProcessType type() const override; - std::string name() const override; - - // grid - void set_grids(const std::shared_ptr grids_manager) override; - - // management of common atm process memory - size_t requested_buffer_size_in_bytes() const override; - void init_buffers(const ATMBufferManager &buffer_manager) override; - - // process behavior - void initialize_impl(const RunType run_type) override; - void run_impl(const double dt) override; - void finalize_impl() override; - - // performs some checks on the tracers group - void set_computed_group_impl(const FieldGroup& group) override; - -private: - - // number of horizontal columns and vertical levels - int ncol_, nlev_; - - // Atmosphere processes often have a pre-processing step that constructs - // required variables from the set of fields stored in the field manager. - // This functor implements this step, which is called during run_impl. - struct Preprocess { - Preprocess() = default; - - KOKKOS_INLINE_FUNCTION - void operator()(const Kokkos::TeamPolicy::member_type& team) const { - const int i = team.league_rank(); // column index - - // Compute vertical layer heights - const auto dz_i = ekat::subview(dz_, i); - auto z_iface_i = ekat::subview(z_iface_, i); - auto z_mid_i = ekat::subview(z_mid_, i); - PF::calculate_z_int(team, nlev_, dz_i, z_surf_, z_iface_i); - team.team_barrier(); - PF::calculate_z_mid(team, nlev_, z_iface_i, z_mid_i); - team.team_barrier(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int k) { - //-------------------------- - // Wet to dry mixing ratios - //-------------------------- - // - // Since tracers from the host model (or AD) are wet mixing ratios, and - // MAM4 expects these tracers in dry mixing ratios, we convert the wet - // mixing ratios to dry mixing ratios for all the tracers. - // - // The function calculate_drymmr_from_wetmmr takes 2 arguments: - // 1. wet mmr - // 2. "wet" water vapor mixing ratio - // - // Units of all tracers become [kg/kg(dry-air)] for mass mixing ratios and - // [#/kg(dry-air)] for number mixing ratios after the following - // conversion. qv is converted to dry mmr in the next parallel for. - q_soag_(i,k) = PF::calculate_drymmr_from_wetmmr(q_soag_(i,k), qv_(i,k)); - q_h2so4_(i,k) = PF::calculate_drymmr_from_wetmmr(q_h2so4_(i,k), qv_(i,k)); - - q_aitken_so4_(i,k) = PF::calculate_drymmr_from_wetmmr(q_aitken_so4_(i,k), qv_(i,k)); - - // convert qv to dry mmr - qv_(i,k) = PF::calculate_drymmr_from_wetmmr(qv_(i,k), qv_(i,k)); - }); - team.team_barrier(); - } // operator() - - // number of horizontal columns and vertical levels - int ncol_, nlev_; - - // height of bottom of atmosphere - Real z_surf_; - - // used for converting between wet and dry mixing ratios - view_1d_int convert_wet_dry_idx_d_; - - // local atmospheric state column variables - view_2d T_mid_; // temperature at grid midpoints [K] - view_2d p_mid_; // total pressure at grid midpoints [Pa] - view_2d qv_; // water vapor mass mixing ratio, not const because it - // must be converted from wet to dry [kg vapor/kg dry air] - view_2d z_mid_; // height at layer midpoints [m] - view_2d z_iface_; // height at layer interfaces [m] - view_2d dz_; // layer thickness [m] - view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] - view_2d cloud_f_; // cloud fraction [-] - view_2d uv_; // updraft velocity [m/s] - view_1d pblh_; // planetary boundary layer height [m] - - // local aerosol-related gases - view_2d q_soag_; // secondary organic aerosol gas [kg gas/kg dry air] - view_2d q_h2so4_; // H2SO3 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d n_aitken_; // aitken mode number mixing ratio [1/kg dry air] - view_2d q_aitken_so4_; // SO4 mass mixing ratio in aitken mode [kg/kg dry air] - - // assigns local variables - void set_variables(const int ncol, const int nlev, const Real z_surf, - const view_1d_int& convert_wet_dry_idx_d, - const view_2d& T_mid, - const view_2d& p_mid, - const view_2d& qv, - const view_2d& z_mid, - const view_2d& z_iface, - const view_2d& dz, - const view_2d& pdel, - const view_1d& pblh, - const view_2d& q_soag, - const view_2d& q_h2so4, - const view_2d& q_aitken_so4) { - ncol_ = ncol; - nlev_ = nlev; - z_surf_ = z_surf; - convert_wet_dry_idx_d_ = convert_wet_dry_idx_d; - T_mid_ = T_mid; - p_mid_ = p_mid; - qv_ = qv; - z_mid_ = z_mid; - z_iface_ = z_iface; - dz_ = dz; - pdel_ = pdel; - pblh_ = pblh; - q_soag_ = q_soag; - q_h2so4_ = q_h2so4; - q_aitken_so4_ = q_aitken_so4; - } // set_variables - }; // MAMMicrophysics::Preprocess - - // Postprocessing functor - struct Postprocess { - Postprocess() = default; - - KOKKOS_INLINE_FUNCTION - void operator()(const Kokkos::TeamPolicy::member_type& team) const { - // After these updates, all tracers are converted from dry mmr to wet mmr - const int i = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int k) { - // Here we convert our dry mmrs back to wet mmrs for EAMxx. - // NOTE: calculate_wetmmr_from_drymmr takes 2 arguments: - // 1. dry mmr - // 2. "dry" water vapor mixing ratio - q_soag_(i,k) = PF::calculate_wetmmr_from_drymmr(q_soag_(i,k), qv_(i,k)); - q_h2so4_(i,k) = PF::calculate_wetmmr_from_drymmr(q_h2so4_(i,k), qv_(i,k)); - - q_aitken_so4_(i,k) = PF::calculate_wetmmr_from_drymmr(q_aitken_so4_(i,k), qv_(i,k)); - - qv_(i,k) = PF::calculate_wetmmr_from_drymmr(qv_(i,k), qv_(i,k)); - }); - team.team_barrier(); - } // operator() - - // Local variables - int ncol_, nlev_; - view_2d qv_; - - // used for converting between wet and dry mixing ratios - view_1d_int convert_wet_dry_idx_d_; - - // local aerosol-related gases - view_2d q_soag_; // secondary organic aerosol gas [kg gas/kg dry air] - view_2d q_h2so4_; // H2SO3 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d q_aitken_so4_; // SO4 aerosol in aitken mode [kg/kg dry air] - - // assigns local variables - void set_variables(const int ncol, - const int nlev, - const view_1d_int& convert_wet_dry_idx_d, - const view_2d& qv, - const view_2d& q_soag, - const view_2d& q_h2so4, - const view_2d& q_aitken_so4) { - ncol_ = ncol; - nlev_ = nlev; - convert_wet_dry_idx_d_ = convert_wet_dry_idx_d; - qv_ = qv; - q_soag_ = q_soag; - q_h2so4_ = q_h2so4; - q_aitken_so4_ = q_aitken_so4; - } // set_variables - }; // MAMMicrophysics::Postprocess - - // storage for local variables, initialized with ATMBufferManager - struct Buffer { - // number of fields stored at column midpoints - static constexpr int num_2d_mid = 2; - - // number of fields stored at column interfaces - static constexpr int num_2d_iface = 1; - - // column midpoint fields - uview_2d z_mid; // height at midpoints - uview_2d dz; // layer thickness - - // column interface fields - uview_2d z_iface; // height at interfaces - - // storage - Real* wsm_data; - }; - - // MAM4 aerosol particle size description - mam4::AeroConfig aero_config_; - - // aerosol processes - std::unique_ptr nucleation_; - - // pre- and postprocessing scratch pads - Preprocess preprocess_; - Postprocess postprocess_; - - // local atmospheric state column variables - view_2d T_mid_; // temperature at grid midpoints [K] - view_2d p_mid_; // total pressure at grid midpoints [Pa] - view_2d qv_; // water vapor mass mixing ratio, not const because it - // must be converted from wet to dry [kg vapor/kg dry air] - view_2d height_; // height at grid interfaces [m] - view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] - view_2d cloud_f_; // cloud fraction [-] - view_2d uv_; // updraft velocity [m/s] - view_1d pblh_; // planetary boundary layer height [m] - - // local aerosol-related gases - view_2d q_soag_; // secondary organic aerosol gas [kg gas/kg dry air] - view_2d q_h2so4_; // H2SO3 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d n_aitken_; // aitken mode number mixing ratio [1/kg dry air] - view_2d q_aitken_so4_; // SO4 mass mixing ratio in aitken mode [kg/kg dry air] - - // workspace manager for internal local variables - //ekat::WorkspaceManager workspace_mgr_; - Buffer buffer_; - - // physics grid for column information - std::shared_ptr grid_; -}; // MAMMicrophysics - -} // namespace scream - -#endif // EAMXX_MAM_MICROPHYSICS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp similarity index 62% rename from components/eamxx/src/physics/mam/eamxx_mam_microphysics.cpp rename to components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 65e358e975da..1a89e1767522 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -21,7 +21,7 @@ AtmosphereProcessType MAMMicrophysics::type() const { } std::string MAMMicrophysics::name() const { - return "MAMMicrophysics"; + return "mam4_micro"; } void MAMMicrophysics::set_grids(const std::shared_ptr grids_manager) { @@ -51,26 +51,31 @@ void MAMMicrophysics::set_grids(const std::shared_ptr grids_ FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // Define fields needed in mam4xx. - const auto m2 = m*m; - const auto s2 = s*s; // atmospheric quantities - add_field("T_mid", scalar3d_layout_mid, K, grid_name); + add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // vertical pressure velocity + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // total pressure - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); - add_field("pbl_height", scalar2d_layout_col, m, grid_name); - add_field("pseudo_density", scalar3d_layout_mid, q_unit, grid_name); // pdel - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // specific humidity + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice wet mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio + add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // pdel, hydrostatic pressure + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction + + // droplet activation can alter cloud liquid and number mixing ratios + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid wet mixing ratio + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // cloud liquid wet number mixing ratio // aerosol tracers of interest: mass (q) and number (n) mixing ratios - add_field("q_aitken_so4", scalar3d_layout_mid, q_unit, grid_name, "tracers"); - add_field("n_aitken_so4", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_field("q_aitken_so4", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // sulfate mixing ratio for aitken mode + add_field("n_aitken", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // number mixing ratio of aitken mode // aerosol-related gases: mass mixing ratios - add_field("q_soag", scalar3d_layout_mid, q_unit, grid_name, "tracers"); - add_field("q_h2so4", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("q_h2so4", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // wet mixing ratio of sulfuric acid gas - // Tracer group -- do we need this in addition to the tracers above? + // Tracers group -- do we need this in addition to the tracers above? In any + // case, this call should be idempotent, so it can't hurt. add_group("tracers", grid_name, 1, Bundling::Required); } @@ -85,8 +90,11 @@ set_computed_group_impl(const FieldGroup& group) { "Error! MAM4 expects bundled fields for tracers.\n"); // How many aerosol/gas tracers do we expect? Recall that we maintain - // both cloudborne and interstitial aerosol tracers. - int num_aero_tracers = 4; // for now, just 2 gases + aitken so4 n,q + // both cloudborne and interstitial aerosol tracers. For now, though, we have + // 3 aerosol tracers: + // * H2SO4 gas mass mixing ratio + // * interstitial aitken-mode sulfate number + mass mixing ratios + int num_aero_tracers = 3; /* aero_config_.num_gas_ids() + // gas tracers 2 * aero_config_.num_modes(); // modal number mixing ratio tracers @@ -108,7 +116,7 @@ set_computed_group_impl(const FieldGroup& group) { size_t MAMMicrophysics::requested_buffer_size_in_bytes() const { // number of Reals needed by local views in interface - const size_t iface_request = sizeof(Real) * + const size_t request = sizeof(Real) * (Buffer::num_2d_mid * ncol_ * nlev_ + Buffer::num_2d_iface * ncol_ * (nlev_+1)); @@ -121,7 +129,7 @@ size_t MAMMicrophysics::requested_buffer_size_in_bytes() const const size_t wsm_request= WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy); */ - return iface_request;// + wsm_request; + return request;// + wsm_request; } void MAMMicrophysics::init_buffers(const ATMBufferManager &buffer_manager) { @@ -132,7 +140,19 @@ void MAMMicrophysics::init_buffers(const ATMBufferManager &buffer_manager) { // set view pointers for midpoint fields using view_2d_t = decltype(buffer_.z_mid); - view_2d_t* view_2d_mid_ptrs[Buffer::num_2d_mid] = {&buffer_.z_mid, &buffer_.dz}; + view_2d_t* view_2d_mid_ptrs[Buffer::num_2d_mid] = { + &buffer_.z_mid, + &buffer_.dz, + &buffer_.qv_dry, + &buffer_.qc_dry, + &buffer_.n_qc_dry, + &buffer_.qi_dry, + &buffer_.n_qi_dry, + &buffer_.w_updraft, + &buffer_.q_h2so4_tend, + &buffer_.n_aitken_tend, + &buffer_.q_aitken_so4_tend, + }; for (int i = 0; i < Buffer::num_2d_mid; ++i) { *view_2d_mid_ptrs[i] = view_2d_t(mem, ncol_, nlev_); mem += view_2d_mid_ptrs[i]->size(); @@ -164,12 +184,21 @@ void MAMMicrophysics::init_buffers(const ATMBufferManager &buffer_manager) { } void MAMMicrophysics::initialize_impl(const RunType run_type) { - const auto& T_mid = get_field_in("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& qv = get_field_in("qv").get_view(); - const auto& pblh = get_field_in("pbl_height").get_view(); - const auto& p_del = get_field_in("pseudo_density").get_view(); - const auto& cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? + + const auto& T_mid = get_field_in("T_mid").get_view(); + const auto& p_mid = get_field_in("p_mid").get_view(); + const auto& qv = get_field_in("qv").get_view(); + const auto& pblh = get_field_in("pbl_height").get_view(); + const auto& p_del = get_field_in("pseudo_density").get_view(); + const auto& cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? + const auto& qc = get_field_out("qc").get_view(); + const auto& n_qc = get_field_out("nc").get_view(); + const auto& qi = get_field_in("qi").get_view(); + const auto& n_qi = get_field_in("ni").get_view(); + const auto& omega = get_field_in("omega").get_view(); + const auto& q_h2so4 = get_field_out("q_h2so4").get_view(); + const auto& n_aitken = get_field_out("n_aitken").get_view(); + const auto& q_aitken_so4 = get_field_out("q_aitken_so4").get_view(); const auto& tracers = get_group_out("tracers"); const auto& tracers_info = tracers.m_info; @@ -179,6 +208,12 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { auto z_mid = buffer_.z_mid; auto dz = buffer_.dz; auto z_iface = buffer_.z_iface; + auto qv_dry = buffer_.qv_dry; + auto qc_dry = buffer_.qc_dry; + auto n_qc_dry = buffer_.n_qc_dry; + auto qi_dry = buffer_.qi_dry; + auto n_qi_dry = buffer_.n_qi_dry; + auto w_updraft = buffer_.w_updraft; // Perform any initialization work. if (run_type==RunType::Initial){ @@ -195,35 +230,41 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { T_mid_ = T_mid; p_mid_ = p_mid; qv_ = qv; + qc_ = qc; + n_qc_ = n_qc; + qi_ = qi; + n_qi_ = n_qi; pdel_ = p_del; -// cloud_f_ = cloud_f; // cloud fraction -// uv_ = uv; // updraft velocity + cloud_f_ = cldfrac; + pblh_ = pblh; + q_h2so4_ = q_h2so4; + q_aitken_so4_ = q_aitken_so4; + n_aitken_ = n_aitken; - // For now, set z_surf to zero. + // FIXME: For now, set z_surf to zero. const Real z_surf = 0.0; // Determine indices of aerosol/gas tracers for wet<->dry conversion auto q_aitken_so4_index = tracers_info->m_subview_idx.at("q_aitken_so4"); - auto q_soag_index = tracers_info->m_subview_idx.at("q_soag"); auto q_h2so4_index = tracers_info->m_subview_idx.at("q_h2so4"); - int num_aero_tracers = 3; // for now, just 2 gases + aitken so4 + int num_aero_tracers = 2; // for now, just 1 gas + aitken so4 view_1d_int convert_wet_dry_idx_d("convert_wet_dry_idx_d", num_aero_tracers); auto convert_wet_dry_idx_h = Kokkos::create_mirror_view(convert_wet_dry_idx_d); for (int it=0, iq=0; it < num_tracers; ++it) { - if ((it == q_aitken_so4_index) || (it == q_soag_index) || (it == q_h2so4_index)) { + if ((it == q_aitken_so4_index) || (it == q_h2so4_index)) { convert_wet_dry_idx_h(iq) = it; ++iq; } } Kokkos::deep_copy(convert_wet_dry_idx_d, convert_wet_dry_idx_h); + // hand views to our preprocess/postprocess functors preprocess_.set_variables(ncol_, nlev_, z_surf, convert_wet_dry_idx_d, T_mid, - p_mid, qv, z_mid, z_iface, dz, pdel_, pblh, q_soag_, - q_h2so4_, q_aitken_so4_); - - // FIXME: here we run the aerosol microphysics parameterizations - - //postprocess_.set_variables(ncol_, nlev_, qv, q_soag, q_h2so4, q_aitken_so4); + p_mid, qv, qv_dry, qc, n_qc, qc_dry, n_qc_dry, qi, n_qi, + qi_dry, n_qi_dry, z_mid, z_iface, dz, pdel_, cldfrac, omega, w_updraft, pblh, + q_h2so4_, q_aitken_so4_, n_aitken_); + postprocess_.set_variables(ncol_, nlev_, convert_wet_dry_idx_d, qv_dry, + q_h2so4_, q_aitken_so4_, n_aitken_); // Set field property checks for the fields in this process /* e.g. @@ -240,7 +281,16 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { //const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); //workspace_mgr_.setup(buffer_.wsm_data, nlev_+1, 13+(n_wind_slots+n_trac_slots), default_policy); - // FIXME: aerosol process initialization goes here! + // configure the nucleation parameterization + mam4::NucleationProcess::ProcessConfig nuc_config{}; + nuc_config.dens_so4a_host = 1770.0; + nuc_config.mw_so4a_host = 115.0; + nuc_config.newnuc_method_user_choice = 2; + nuc_config.pbl_nuc_wang2008_user_choice = 1; + nuc_config.adjust_factor_pbl_ratenucl = 1.0; + nuc_config.accom_coef_h2so4 = 1.0; + nuc_config.newnuc_adjust_factor_dnaitdt = 1.0; + nucleation_->init(nuc_config); } void MAMMicrophysics::run_impl(const double dt) { @@ -255,14 +305,21 @@ void MAMMicrophysics::run_impl(const double dt) { // Reset internal WSM variables. //workspace_mgr_.reset_internals(); - // nothing depends on simulation time (yet), so we can just use zero for now + // FIXME: nothing depends on simulation time (yet), so we can just use zero for now double t = 0.0; - // FIXME: for now, we set cloud fraction and updraft velocity to zero. - view_1d cloud_f("cloud fraction", nlev_); - view_1d uv("updraft velocity", nlev_); - Kokkos::deep_copy(cloud_f, 0.0); - Kokkos::deep_copy(uv, 0.0); + // Alias member variables + auto T_mid = T_mid_; + auto p_mid = p_mid_; + auto qv_dry = buffer_.qv_dry; + auto qc = qc_; + auto n_qc = n_qc_; + auto qi = qi_; + auto n_qi = n_qi_; + auto z_mid = buffer_.z_mid; + auto cldfrac = cloud_f_; + auto pdel = pdel_; + auto w_updraft = buffer_.w_updraft; // Compute nucleation tendencies on all local columns and accumulate them // into our tracer state. @@ -270,17 +327,23 @@ void MAMMicrophysics::run_impl(const double dt) { const Int icol = team.league_rank(); // column index // extract column-specific atmosphere state data - haero::Atmosphere atm(nlev_, pblh_(icol)); - atm.temperature = ekat::subview(T_mid_, icol); - atm.pressure = ekat::subview(p_mid_, icol); - atm.vapor_mixing_ratio = ekat::subview(qv_, icol); - atm.height = ekat::subview(height_, icol); - atm.hydrostatic_dp = ekat::subview(pdel_, icol); - atm.cloud_fraction = cloud_f; - atm.updraft_vel_ice_nucleation = uv; + haero::Atmosphere atm(nlev_, ekat::subview(T_mid_, icol), + ekat::subview(p_mid_, icol), + ekat::subview(qv_dry, icol), + ekat::subview(qc_, icol), + ekat::subview(n_qc_, icol), + ekat::subview(qi_, icol), + ekat::subview(n_qi_, icol), + ekat::subview(z_mid, icol), + ekat::subview(pdel, icol), + ekat::subview(cldfrac, icol), + ekat::subview(w_updraft, icol), + pblh_(icol)); + + // set surface state data + haero::Surface sfc{}; // extract column-specific subviews into aerosol prognostics - using AeroConfig = mam4::AeroConfig; using ModeIndex = mam4::ModeIndex; using AeroId = mam4::AeroId; using GasId = mam4::GasId; @@ -293,17 +356,31 @@ void MAMMicrophysics::run_impl(const double dt) { int ih2so4 = static_cast(GasId::H2SO4); progs.q_gas[ih2so4] = ekat::subview(q_h2so4_, icol); + // nucleation doesn't use any diagnostics, so it's okay to leave this alone + // for now mam4::Diagnostics diags(nlev_); - // run the nucleation process to obtain tendencies + // grab views from the buffer to store tendencies mam4::Tendencies tends(nlev_); - nucleation_->compute_tendencies(team, t, dt, atm, progs, diags, tends); + tends.q_gas[ih2so4] = ekat::subview(buffer_.q_h2so4_tend, icol); + tends.n_mode_i[iait] = ekat::subview(buffer_.n_aitken_tend, icol); + tends.q_aero_i[iait][iso4] = ekat::subview(buffer_.q_aitken_so4_tend, icol); - // accumulate tendencies into tracers state + // run the nucleation process to obtain tendencies + nucleation_->compute_tendencies(team, t, dt, atm, sfc, progs, diags, tends); +#ifndef NDEBUG + const int lev_idx = 0; + if (icol == 0) { + m_atm_logger->debug("tends.q_gas[ih2so4] = {}, tends.n_mode_i[iait] = {}, tends.q_aero_i[iait][iso4] = {}", + tends.q_gas[ih2so4](lev_idx), tends.n_mode_i[iait](lev_idx), tends.q_aero_i[iait][iso4](lev_idx)); + } +#endif + + // accumulate tendencies into prognostics Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int klev) { + progs.q_gas[ih2so4](klev) += dt * tends.q_gas[ih2so4](klev); progs.n_mode_i[iait](klev) += dt * tends.n_mode_i[iait](klev); progs.q_aero_i[iait][iso4](klev) += dt * tends.q_aero_i[iait][iso4](klev); - progs.q_gas[ih2so4](klev) += dt * tends.q_gas[ih2so4](klev); }); }); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp new file mode 100644 index 000000000000..ff3652459f1e --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -0,0 +1,365 @@ +#ifndef EAMXX_MAM_MICROPHYSICS_HPP +#define EAMXX_MAM_MICROPHYSICS_HPP + +#include +#include +#include + +#include +#include +#include + +#include + +#ifndef KOKKOS_ENABLE_CUDA +#define protected_except_cuda public +#define private_except_cuda public +#else +#define protected_except_cuda protected +#define private_except_cuda private +#endif + +namespace scream +{ + +// The process responsible for handling MAM4 aerosols. The AD stores exactly ONE +// instance of this class in its list of subcomponents. +class MAMMicrophysics final : public scream::AtmosphereProcess { + using PF = scream::PhysicsFunctions; + using KT = ekat::KokkosTypes; + + // views for single- and multi-column data + using view_1d_int = typename KT::template view_1d; + using view_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; + using const_view_1d = typename KT::template view_1d; + using const_view_2d = typename KT::template view_2d; + + // unmanaged views (for buffer and workspace manager) + using uview_1d = Unmanaged>; + using uview_2d = Unmanaged>; + + // a quantity stored in a single vertical column with a single index + using ColumnView = mam4::ColumnView; + + // a thread team dispatched to a single vertical column + using ThreadTeam = mam4::ThreadTeam; + +public: + + // Constructor + MAMMicrophysics(const ekat::Comm& comm, const ekat::ParameterList& params); + +protected_except_cuda: + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // process metadata + AtmosphereProcessType type() const override; + std::string name() const override; + + // grid + void set_grids(const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // process behavior + void initialize_impl(const RunType run_type) override; + void run_impl(const double dt) override; + void finalize_impl() override; + + // performs some checks on the tracers group + void set_computed_group_impl(const FieldGroup& group) override; + +private_except_cuda: + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + + KOKKOS_INLINE_FUNCTION + void operator()(const Kokkos::TeamPolicy::member_type& team) const { + const int i = team.league_rank(); // column index + + // Compute vertical layer heights + const auto dz_i = ekat::subview(dz_, i); + auto z_iface_i = ekat::subview(z_iface_, i); + auto z_mid_i = ekat::subview(z_mid_, i); + PF::calculate_z_int(team, nlev_, dz_i, z_surf_, z_iface_i); + team.team_barrier(); // TODO: is this barrier necessary? + PF::calculate_z_mid(team, nlev_, z_iface_i, z_mid_i); + // barrier here allows the kernels that follow to use layer heights + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_), + [&] (const int k) { + //-------------------------- + // Vertical velocity from pressure to height + //-------------------------- + const auto rho = PF::calculate_density(pdel_(i,k), dz_i(k)); + w_updraft_(i,k) = PF::calculate_vertical_velocity(omega_(i,k), rho); + + //-------------------------- + // Wet to dry mixing ratios + //-------------------------- + // + // Since tracers from the host model (or AD) are wet mixing ratios, and + // MAM4 expects these tracers in dry mixing ratios, we convert the wet + // mixing ratios to dry mixing ratios for all the tracers. + // + // The function calculate_drymmr_from_wetmmr takes 2 arguments: + // 1. wet mmr + // 2. "wet" water vapor mixing ratio + // + // Units of all tracers become [kg/kg(dry-air)] for mass mixing ratios and + // [#/kg(dry-air)] for number mixing ratios after the following + // conversion. + const auto qv_ik = qv_(i,k); + // const fields need separate storage for "dry" values + qv_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qv_(i,k), qv_ik); + qc_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qc_(i,k), qv_ik); + n_qc_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(n_qc_(i,k), qv_ik); + qi_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qi_(i,k), qv_ik); + n_qi_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(n_qi_(i,k), qv_ik); + + // non-const fields can be overwritten; we'll convert back to moist + // air ratios during postprocess + q_h2so4_(i,k) = PF::calculate_drymmr_from_wetmmr(q_h2so4_(i,k), qv_ik); + q_aitken_so4_(i,k) = PF::calculate_drymmr_from_wetmmr(q_aitken_so4_(i,k), qv_ik); + n_aitken_(i,k) = PF::calculate_drymmr_from_wetmmr(n_aitken_(i,k), qv_ik); + }); + // make sure all operations are done before exiting kernel + team.team_barrier(); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // height of bottom of atmosphere + Real z_surf_; + + // used for converting between wet and dry mixing ratios + view_1d_int convert_wet_dry_idx_d_; + + // local atmospheric state column variables + const_view_2d T_mid_; // temperature at grid midpoints [K] + const_view_2d p_mid_; // total pressure at grid midpoints [Pa] + const_view_2d qv_; // water vapor specific humidity [kg vapor / kg moist air] + view_2d qv_dry_; // water vapor mixing ratio [kg vapor / kg dry air] + const_view_2d qc_; // cloud liquid water mass mixing ratio [kg vapor/kg moist air] + view_2d qc_dry_; + const_view_2d n_qc_; // cloud liquid water number mixing ratio [kg cloud water / kg moist air] + view_2d n_qc_dry_; // cloud liquid water number mixing ratio (dry air) + const_view_2d qi_; // cloud ice water mass mixing ratio + view_2d qi_dry_; // [kg vapor/kg dry air] + const_view_2d n_qi_; // cloud ice water number mixing ratio + view_2d n_qi_dry_; + view_2d z_mid_; // height at layer midpoints [m] + view_2d z_iface_; // height at layer interfaces [m] + view_2d dz_; // layer thickness [m] + const_view_2d pdel_; // hydrostatic "pressure thickness" at grid + // interfaces [Pa] + const_view_2d cloud_f_; // cloud fraction [-] + const_view_2d omega_; // vertical pressure velocity [Pa/s] + view_2d w_updraft_; // updraft velocity [m/s] + const_view_1d pblh_; // planetary boundary layer height [m] + + // local aerosol-related gases + view_2d q_h2so4_; // H2SO4 gas [kg/kg dry air] + + // local aerosols (more to appear as we improve this atm process) + view_2d n_aitken_; // aitken mode number mixing ratio [1/kg dry air] + view_2d q_aitken_so4_; // SO4 mass mixing ratio in aitken mode [kg/kg dry air] + + // assigns local variables + void set_variables(const int ncol, const int nlev, const Real z_surf, + const view_1d_int& convert_wet_dry_idx_d, + const const_view_2d& T_mid, + const const_view_2d& p_mid, + const const_view_2d& qv, + const view_2d& qv_dry, + const view_2d& qc, + const view_2d& n_qc, + const view_2d& qc_dry, + const view_2d& n_qc_dry, + const const_view_2d& qi, + const const_view_2d& n_qi, + const view_2d& qi_dry, + const view_2d& n_qi_dry, + const view_2d& z_mid, + const view_2d& z_iface, + const view_2d& dz, + const const_view_2d& pdel, + const const_view_2d& cf, + const const_view_2d& omega, + const view_2d& w_updraft, + const const_view_1d& pblh, + const view_2d& q_h2so4, + const view_2d& q_aitken_so4, + const view_2d& n_aitken) { + ncol_ = ncol; + nlev_ = nlev; + z_surf_ = z_surf; + convert_wet_dry_idx_d_ = convert_wet_dry_idx_d; + T_mid_ = T_mid; + p_mid_ = p_mid; + qv_ = qv; + qv_dry_ = qv_dry; + qc_ = qc; + n_qc_ = n_qc; + qc_dry_ = qc_dry; + n_qc_dry_ = n_qc_dry; + qi_ = qi; + n_qi_ = n_qi; + qi_dry_ = qi_dry; + n_qi_dry_ = n_qi_dry; + z_mid_ = z_mid; + z_iface_ = z_iface; + dz_ = dz; + pdel_ = pdel; + cloud_f_ = cf; + omega_ = omega; + w_updraft_ = w_updraft; + pblh_ = pblh; + q_h2so4_ = q_h2so4; + q_aitken_so4_ = q_aitken_so4; + n_aitken_ = n_aitken; + } // set_variables + }; // MAMMicrophysics::Preprocess + + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + KOKKOS_INLINE_FUNCTION + void operator()(const Kokkos::TeamPolicy::member_type& team) const { + // After these updates, all non-const tracers are converted from dry mmr to wet mmr + const int i = team.league_rank(); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_), + [&] (const int k) { + const auto qv_ik = qv_dry_(i,k); + q_h2so4_(i,k) = PF::calculate_wetmmr_from_drymmr(q_h2so4_(i,k), qv_ik); + q_aitken_so4_(i,k) = PF::calculate_wetmmr_from_drymmr(q_aitken_so4_(i,k), qv_ik); + n_aitken_(i,k) = PF::calculate_wetmmr_from_drymmr(n_aitken_(i,k), qv_ik); + }); + team.team_barrier(); + } // operator() + + // Local variables + int ncol_, nlev_; + view_2d qv_dry_; + + // used for converting between wet and dry mixing ratios + view_1d_int convert_wet_dry_idx_d_; + + // local aerosol-related gases + view_2d q_h2so4_; // H2SO4 gas [kg/kg dry air] + + // local aerosols (more to appear as we improve this atm process) + view_2d q_aitken_so4_; // SO4 aerosol in aitken mode [kg/kg dry air] + + // modal quantities + view_2d n_aitken_; + + // assigns local variables + void set_variables(const int ncol, + const int nlev, + const view_1d_int& convert_wet_dry_idx_d, + const view_2d& qv_dry, + const view_2d& q_h2so4, + const view_2d& q_aitken_so4, + const view_2d& n_aitken) { + ncol_ = ncol; + nlev_ = nlev; + convert_wet_dry_idx_d_ = convert_wet_dry_idx_d; + qv_dry_ = qv_dry; + q_h2so4_ = q_h2so4; + q_aitken_so4_ = q_aitken_so4; + n_aitken_ = n_aitken; + } // set_variables + }; // MAMMicrophysics::Postprocess + + // storage for local variables, initialized with ATMBufferManager + struct Buffer { + // number of local fields stored at column midpoints + static constexpr int num_2d_mid = 11; + + // local column midpoint fields + uview_2d z_mid; // height at midpoints + uview_2d dz; // layer thickness + uview_2d qv_dry; // water vapor mixing ratio (dry air) + uview_2d qc_dry; // cloud water mass mixing ratio + uview_2d n_qc_dry; // cloud water number mixing ratio + uview_2d qi_dry; // cloud ice mass mixing ratio + uview_2d n_qi_dry; // cloud ice number mixing ratio + uview_2d w_updraft; // vertical wind velocity + uview_2d q_h2so4_tend; // tendency for H2SO4 gas + uview_2d n_aitken_tend; // tendency for aitken aerosol mode + uview_2d q_aitken_so4_tend; // tendency for aitken mode sulfate aerosol + + // number of local fields stored at column interfaces + static constexpr int num_2d_iface = 1; + + // local column interface fields + uview_2d z_iface; // height at interfaces + + // storage + Real* wsm_data; + }; + + // MAM4 aerosol particle size description + mam4::AeroConfig aero_config_; + + // aerosol processes + std::unique_ptr nucleation_; + + // pre- and postprocessing scratch pads + Preprocess preprocess_; + Postprocess postprocess_; + + // local atmospheric state column variables + const_view_2d T_mid_; // temperature at grid midpoints [K] + const_view_2d p_mid_; // total pressure at grid midpoints [Pa] + const_view_2d qv_; // water vapor specific humidity [kg h2o vapor / kg moist air] + // we keep the specific humidity to use in the PostProcess step. + view_2d qc_; // cloud liquid mass mixing ratio + // must be converted from wet to dry [kg cloud water /kg dry air] + view_2d n_qc_; // cloud liquid number mixing ratio + // must be converted from wet to dry [1 /kg dry air] + const_view_2d qi_; // cloud ice mass mixing ratio + // must be converted from wet to dry [kg cloud water /kg dry air] + const_view_2d n_qi_; // cloud ice number mixing ratio + // must be converted from wet to dry [1 /kg dry air] + const_view_2d pdel_; // hydrostatic "pressure thickness" at grid + // interfaces [Pa] + const_view_2d cloud_f_; // cloud fraction [-] + const_view_1d pblh_; // planetary boundary layer height [m] + + // local aerosol-related gases + view_2d q_h2so4_; // H2SO3 gas [kg/kg dry air] + + // local aerosols (more to appear as we improve this atm process) + view_2d n_aitken_; // aitken mode number mixing ratio [1/kg dry air] + view_2d q_aitken_so4_; // SO4 mass mixing ratio in aitken mode [kg/kg dry air] + + // workspace manager for internal local variables + //ekat::WorkspaceManager workspace_mgr_; + Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; +}; // MAMMicrophysics + +} // namespace scream + +#endif // EAMXX_MAM_MICROPHYSICS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp new file mode 100644 index 000000000000..85b51ff3f3dc --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -0,0 +1,107 @@ +#include +#include +#include + +#include "scream_config.h" // for SCREAM_CIME_BUILD + +#include + +namespace scream +{ + +MAMOptics::MAMOptics( + const ekat::Comm& comm, + const ekat::ParameterList& params) + : AtmosphereProcess(comm, params), + aero_config_() { +} + +AtmosphereProcessType MAMOptics::type() const { + return AtmosphereProcessType::Physics; +} + +std::string MAMOptics::name() const { + return "mam4_optics"; +} + +void MAMOptics::set_grids(const std::shared_ptr grids_manager) { + using namespace ekat::units; + + grid_ = grids_manager->get_grid("Physics"); + const auto& grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // number of levels per column + nswbands_ = 14; // number of shortwave bands + nlwbands_ = 16; // number of longwave bands + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Define aerosol optics fields computed by this process. + auto nondim = Units::nondimensional(); + FieldLayout scalar3d_swband_layout { {COL, SWBND, LEV}, {ncol_, nswbands_, nlev_} }; + FieldLayout scalar3d_lwband_layout { {COL, LWBND, LEV}, {ncol_, nlwbands_, nlev_} }; + + // shortwave aerosol scattering asymmetry parameter [-] + add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); + // shortwave aerosol single-scattering albedo [-] + add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); + // shortwave aerosol optical depth [-] + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + // longwave aerosol optical depth [-] + add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); + + // FIXME: this field doesn't belong here, but this is a convenient place to + // FIXME: put it for now. + // number mixing ratio for CCN + using Spack = ekat::Pack; + using Pack = ekat::Pack; + constexpr int ps = Pack::n; + FieldLayout scalar3d_layout_mid { {COL, LEV}, {ncol_, nlev_} }; + add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); +} + +void MAMOptics::initialize_impl(const RunType run_type) { +} + +void MAMOptics::run_impl(const double dt) { + + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + + // get the aerosol optics fields + auto aero_g_sw = get_field_out("aero_g_sw").get_view(); + auto aero_ssa_sw = get_field_out("aero_ssa_sw").get_view(); + auto aero_tau_sw = get_field_out("aero_tau_sw").get_view(); + auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); + + auto aero_nccn = get_field_out("nccn").get_view(); // FIXME: get rid of this + + // Compute optical properties on all local columns. + // (Strictly speaking, we don't need this parallel_for here yet, but we leave + // it in anticipation of column-specific aerosol optics to come.) + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { + const Int icol = team.league_rank(); // column index + + auto g_sw = ekat::subview(aero_g_sw, icol); + auto ssa_sw = ekat::subview(aero_ssa_sw, icol); + auto tau_sw = ekat::subview(aero_tau_sw, icol); + auto tau_lw = ekat::subview(aero_tau_lw, icol); + + // populate these fields with reasonable representative values + Kokkos::deep_copy(g_sw, 0.5); + Kokkos::deep_copy(ssa_sw, 0.7); + Kokkos::deep_copy(tau_sw, 0.0); + Kokkos::deep_copy(tau_lw, 0.0); + + // FIXME: Get rid of this + auto nccn = ekat::subview(aero_nccn, icol); + Kokkos::deep_copy(nccn, 50.0); + }); +} + +void MAMOptics::finalize_impl() +{ +} + +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp new file mode 100644 index 000000000000..ebd69437dee1 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp @@ -0,0 +1,80 @@ +#ifndef EAMXX_MAM_OPTICS_HPP +#define EAMXX_MAM_OPTICS_HPP + +#include +#include +#include + +#include +#include +#include + +#include + +#ifndef KOKKOS_ENABLE_CUDA +#define protected_except_cuda public +#define private_except_cuda public +#else +#define protected_except_cuda protected +#define private_except_cuda private +#endif + +namespace scream +{ + +// The process responsible for handling MAM4 aerosol optical properties. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMOptics final : public scream::AtmosphereProcess { + using PF = scream::PhysicsFunctions; + using KT = ekat::KokkosTypes; + + // a quantity stored in a single vertical column with a single index + using ColumnView = mam4::ColumnView; + + // a thread team dispatched to a single vertical column + using ThreadTeam = mam4::ThreadTeam; + +public: + + // Constructor + MAMOptics(const ekat::Comm& comm, const ekat::ParameterList& params); + +protected_except_cuda: + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // process metadata + AtmosphereProcessType type() const override; + std::string name() const override; + + // grid + void set_grids(const std::shared_ptr grids_manager) override; + + // process behavior + void initialize_impl(const RunType run_type) override; + void run_impl(const double dt) override; + void finalize_impl() override; + +private_except_cuda: + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // number of shortwave and longwave radiation bands + int nswbands_, nlwbands_; + + // MAM4 aerosol particle size description + mam4::AeroConfig aero_config_; + + // aerosol processes + //std::unique_ptr optics_; + + // physics grid for column information + std::shared_ptr grid_; +}; // MAMOptics + +} // namespace scream + +#endif // EAMXX_MAM_OPTICS_HPP diff --git a/components/eamxx/src/physics/ml_correction/CMakeLists.txt b/components/eamxx/src/physics/ml_correction/CMakeLists.txt index e5d006bc7bb7..d5c94a58d3cf 100644 --- a/components/eamxx/src/physics/ml_correction/CMakeLists.txt +++ b/components/eamxx/src/physics/ml_correction/CMakeLists.txt @@ -1,12 +1,24 @@ set(MLCORRECTION_SRCS - atmosphere_ml_correction.cpp + eamxx_ml_correction_process_interface.cpp ) set(MLCORRECTION_HEADERS - atmosphere_ml_correction.hpp + eamxx_ml_correction_process_interface.hpp ) +include(ScreamUtils) + if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.11.0") + message(STATUS "Downloading Pybind11") + include(FetchContent) + + FetchContent_Declare(pybind11 GIT_REPOSITORY https://github.com/pybind/pybind11.git GIT_TAG v2.10.4) + FetchContent_MakeAvailable(pybind11) +else() + message(FATAL_ERROR "pybind11 is missing. Use CMake >= 3.11 or download it") +endif() +find_package(Python REQUIRED COMPONENTS Interpreter Development) add_library(ml_correction ${MLCORRECTION_SRCS}) -target_include_directories(ml_correction PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../share) -target_link_libraries(ml_correction physics_share scream_share) -target_compile_options(ml_correction PUBLIC) +target_compile_definitions(ml_correction PUBLIC EAMXX_HAS_ML_CORRECTION) +target_compile_definitions(ml_correction PRIVATE -DML_CORRECTION_CUSTOM_PATH="${CMAKE_CURRENT_SOURCE_DIR}") +target_include_directories(ml_correction SYSTEM PUBLIC ${PYTHON_INCLUDE_DIRS}) +target_link_libraries(ml_correction physics_share scream_share pybind11::pybind11 Python::Python) diff --git a/components/eamxx/src/physics/ml_correction/atmosphere_ml_correction.cpp b/components/eamxx/src/physics/ml_correction/atmosphere_ml_correction.cpp deleted file mode 100644 index 6672659115b1..000000000000 --- a/components/eamxx/src/physics/ml_correction/atmosphere_ml_correction.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include - -#include "atmosphere_ml_correction.hpp" -#include "ekat/ekat_assert.hpp" -#include "ekat/util/ekat_units.hpp" - -namespace scream { -// ========================================================================================= -MLCorrection::MLCorrection(const ekat::Comm &comm, - const ekat::ParameterList ¶ms) - : AtmosphereProcess(comm, params) { - // Nothing to do here -} - -// ========================================================================================= -void MLCorrection::set_grids( - const std::shared_ptr grids_manager) { - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Q = kg / kg; - Q.set_string("kg/kg"); - - m_grid = grids_manager->get_grid("Physics"); - const auto &grid_name = m_grid->name(); - m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = - m_grid->get_num_vertical_levels(); // Number of levels per column - - // Define the different field layouts that will be used for this process - - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and - // interfaces - FieldLayout scalar3d_layout_mid{{COL, LEV}, {m_num_cols, m_num_levs}}; - - // Set of fields used strictly as input - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers"); - - // Set of fields used strictly as output - add_field("qv_nudging_tend", scalar3d_layout_mid, Q, grid_name); - - // Set of fields used as input and output - // - There are no fields used as both input and output. -} - -// ========================================================================================= -void MLCorrection::initialize_impl(const RunType /* run_type */) { - // Nothing to do -} - -// ========================================================================================= -void MLCorrection::run_impl(const double /* dt */) { - auto qv = get_field_in("qv").get_view(); - auto qv_nudging = get_field_out("qv_nudging").get_view(); - - // ML correction proceess is not yet implemented -} - -// ========================================================================================= -void MLCorrection::finalize_impl() { - // Do nothing -} -// ========================================================================================= - -} // namespace scream diff --git a/components/eamxx/src/physics/ml_correction/atmosphere_ml_correction.hpp b/components/eamxx/src/physics/ml_correction/atmosphere_ml_correction.hpp deleted file mode 100644 index 5a506646d93d..000000000000 --- a/components/eamxx/src/physics/ml_correction/atmosphere_ml_correction.hpp +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef SCREAM_ML_NUDGING_HPP -#define SCREAM_ML_NUDGING_HPP - -#include - -#include "ekat/ekat_parameter_list.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -namespace scream { - -/* - * The class responsible to handle the calculation of the subgrid cloud - * fractions - * - * The AD should store exactly ONE instance of this class stored - * in its list of subcomponents (the AD should make sure of this). - */ - -class MLCorrection : public AtmosphereProcess { - public: - // Constructors - MLCorrection(const ekat::Comm &comm, const ekat::ParameterList ¶ms); - - // The type of subcomponent - AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } - - // The name of the subcomponent - std::string name() const { return "Machine Learning Nudging"; } - - // Set the grid - void set_grids(const std::shared_ptr grids_manager); - - protected: - // The three main overrides for the subcomponent - void initialize_impl(const RunType run_type); - void run_impl(const double dt); - void finalize_impl(); - - // Keep track of field dimensions and the iteration count - Int m_num_cols; - Int m_num_levs; - - std::shared_ptr m_grid; -}; // class MLCorrection - -} // namespace scream - -#endif // SCREAM_ML_NUDGING_HPP diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp new file mode 100644 index 000000000000..2802e1a590d6 --- /dev/null +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -0,0 +1,146 @@ +#include "eamxx_ml_correction_process_interface.hpp" +#include "ekat/ekat_assert.hpp" +#include "ekat/util/ekat_units.hpp" +#include "share/field/field_utils.hpp" + +namespace scream { +// ========================================================================================= +MLCorrection::MLCorrection(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + m_ML_model_path_tq = m_params.get("ML_model_path_tq"); + m_ML_model_path_uv = m_params.get("ML_model_path_uv"); + m_ML_model_path_sfc_fluxes = m_params.get("ML_model_path_sfc_fluxes"); + m_fields_ml_output_variables = m_params.get>("ML_output_fields"); + m_ML_correction_unit_test = m_params.get("ML_correction_unit_test"); +} + +// ========================================================================================= +void MLCorrection::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto Q = kg / kg; + Q.set_string("kg/kg"); + constexpr int ps = Pack::n; + m_grid = grids_manager->get_grid("Physics"); + const auto &grid_name = m_grid->name(); + m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = + m_grid->get_num_vertical_levels(); // Number of levels per column + // Define the different field layouts that will be used for this process + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + FieldLayout scalar2d_layout{ {COL}, {m_num_cols}}; + FieldLayout scalar3d_layout_mid{{COL, LEV}, {m_num_cols, m_num_levs}}; + FieldLayout scalar3d_layout_int{{COL, ILEV}, {m_num_cols, m_num_levs+1}}; + FieldLayout horiz_wind_layout { {COL,CMP,LEV}, {m_num_cols,2,m_num_levs} }; + if (not m_ML_correction_unit_test) { + const auto m2 = m*m; + const auto s2 = s*s; + auto Wm2 = W / m / m; + auto nondim = m/m; + add_field("phis", scalar2d_layout, m2/s2, grid_name); + add_field("SW_flux_dn", scalar3d_layout_int, Wm2, grid_name, ps); + add_field("sfc_alb_dif_vis", scalar2d_layout, nondim, grid_name); + add_field("sfc_flux_sw_net", scalar2d_layout, Wm2, grid_name); + add_field("sfc_flux_lw_dn", scalar2d_layout, Wm2, grid_name); + m_lat = m_grid->get_geometry_data("lat"); + m_lon = m_grid->get_geometry_data("lon"); + } + + /* ----------------------- WARNING --------------------------------*/ + /* The following is a HACK to get things moving, we don't want to + * add all fields as "updated" long-term. A separate stream of work + * is adapting the infrastructure to allow for a generic "add_field" call + * to be used here which we can then setup using the m_fields_ml_output_variables variable + */ + add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); + add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); + add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + /* ----------------------- WARNING --------------------------------*/ + add_group("tracers", grid_name, 1, Bundling::Required); +} + +void MLCorrection::initialize_impl(const RunType /* run_type */) { + fpe_mask = ekat::get_enabled_fpes(); + ekat::disable_all_fpes(); // required for importing numpy + if ( Py_IsInitialized() == 0 ) { + pybind11::initialize_interpreter(); + } + pybind11::module sys = pybind11::module::import("sys"); + sys.attr("path").attr("insert")(1, ML_CORRECTION_CUSTOM_PATH); + py_correction = pybind11::module::import("ml_correction"); + ML_model_tq = py_correction.attr("get_ML_model")(m_ML_model_path_tq); + ML_model_uv = py_correction.attr("get_ML_model")(m_ML_model_path_uv); + ML_model_sfc_fluxes = py_correction.attr("get_ML_model")(m_ML_model_path_sfc_fluxes); + ekat::enable_fpes(fpe_mask); +} + +// ========================================================================================= +void MLCorrection::run_impl(const double dt) { + // use model time to infer solar zenith angle for the ML prediction + auto current_ts = timestamp(); + std::string datetime_str = current_ts.get_date_string() + " " + current_ts.get_time_string(); + const auto &qv = get_field_out("qv").get_view(); + const auto &T_mid = get_field_out("T_mid").get_view(); + const auto &phis = get_field_in("phis").get_view(); + const auto &SW_flux_dn = get_field_out("SW_flux_dn").get_view(); + const auto &sfc_alb_dif_vis = get_field_in("sfc_alb_dif_vis").get_view(); + const auto &sfc_flux_sw_net = get_field_out("sfc_flux_sw_net").get_view(); + const auto &sfc_flux_lw_dn = get_field_out("sfc_flux_lw_dn").get_view(); + const auto &u = get_field_out("horiz_winds").get_component(0).get_view(); + const auto &v = get_field_out("horiz_winds").get_component(1).get_view(); + + auto h_lat = m_lat.get_view(); + auto h_lon = m_lon.get_view(); + + const auto& tracers = get_group_out("tracers"); + const auto& tracers_info = tracers.m_info; + Int num_tracers = tracers_info->size(); + + ekat::disable_all_fpes(); // required for importing numpy + if ( Py_IsInitialized() == 0 ) { + pybind11::initialize_interpreter(); + } + // for qv, we need to stride across number of tracers + pybind11::object ob1 = py_correction.attr("update_fields")( + pybind11::array_t( + m_num_cols * m_num_levs, T_mid.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols * m_num_levs * num_tracers, qv.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols * m_num_levs, u.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols * m_num_levs, v.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols, h_lat.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols, h_lon.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols, phis.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols * (m_num_levs+1), SW_flux_dn.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols, sfc_alb_dif_vis.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols, sfc_flux_sw_net.data(), pybind11::str{}), + pybind11::array_t( + m_num_cols, sfc_flux_lw_dn.data(), pybind11::str{}), + m_num_cols, m_num_levs, num_tracers, dt, + ML_model_tq, ML_model_uv, ML_model_sfc_fluxes, datetime_str); + pybind11::gil_scoped_release no_gil; + ekat::enable_fpes(fpe_mask); +} + +// ========================================================================================= +void MLCorrection::finalize_impl() { + // Do nothing +} +// ========================================================================================= + +} // namespace scream diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.hpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.hpp new file mode 100644 index 000000000000..eaf5a98b9d33 --- /dev/null +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.hpp @@ -0,0 +1,72 @@ +#ifndef SCREAM_ML_CORRECTION_HPP +#define SCREAM_ML_CORRECTION_HPP + +#include +#include +#include +#include +#include +#include "share/atm_process/atmosphere_process.hpp" +#include "ekat/ekat_parameter_list.hpp" +#include "ekat/util/ekat_lin_interp.hpp" +#include "share/io/scream_output_manager.hpp" +#include "share/io/scorpio_output.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/grid/point_grid.hpp" +#include "share/util/scream_time_stamp.hpp" + +namespace scream { + +/* + * The class responsible to handle the calculation of the subgrid cloud + * fractions + * + * The AD should store exactly ONE instance of this class stored + * in its list of subcomponents (the AD should make sure of this). + */ + +class MLCorrection : public AtmosphereProcess { + public: + using Pack = ekat::Pack; + // Constructors + MLCorrection(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "MLCorrection"; } + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: + // The three main overrides for the subcomponent + void initialize_impl(const RunType run_type); + void run_impl(const double dt); + void finalize_impl(); + void apply_tendency(Field& base, const Field& next, const int dt); + + std::shared_ptr m_grid; + // Keep track of field dimensions and the iteration count + Int m_num_cols; + Int m_num_levs; + Field m_lat; + Field m_lon; + std::string m_ML_model_path_tq; + std::string m_ML_model_path_uv; + std::string m_ML_model_path_sfc_fluxes; + std::vector m_fields_ml_output_variables; + bool m_ML_correction_unit_test; + pybind11::module py_correction; + pybind11::object ML_model_tq; + pybind11::object ML_model_uv; + pybind11::object ML_model_sfc_fluxes; + int fpe_mask; +}; // class MLCorrection + +} // namespace scream + +#endif // SCREAM_ML_CORRECTION_HPP diff --git a/components/eamxx/src/physics/ml_correction/ml_correction.py b/components/eamxx/src/physics/ml_correction/ml_correction.py new file mode 100644 index 000000000000..4e22f711c5ae --- /dev/null +++ b/components/eamxx/src/physics/ml_correction/ml_correction.py @@ -0,0 +1,207 @@ +import numpy as np +import xarray as xr +import datetime +from vcm import cos_zenith_angle +from scream_run.steppers.machine_learning import ( + MachineLearningConfig, + open_model, + predict, +) + + +def get_ML_model(model_path): + if model_path == "NONE": + return None + config = MachineLearningConfig(models=[model_path]) + model = open_model(config) + return model + + +def ensure_correction_ordering(correction): + """Ensure that the ordering of the correction is always (ncol, z)""" + for key in correction: + if "z" in correction[key].dims: + correction[key] = correction[key].transpose("ncol", "z") + return correction + + +def get_ML_correction_dQ1_dQ2(model, T_mid, qv, cos_zenith, dt): + """Get ML correction for air temperature (dQ1) and specific humidity (dQ2) + + Args: + model: pre-trained ML model for dQ1 and dQ2 + T_mid: air temperature + qv: specific humidity + cos_zenith: cosine zenith angle + dt: time step (s) + """ + ds = xr.Dataset( + data_vars=dict( + T_mid=(["ncol", "z"], T_mid), + qv=(["ncol", "z"], qv), + cos_zenith_angle=(["ncol"], cos_zenith), + ) + ) + return ensure_correction_ordering(predict(model, ds, dt)) + + +def get_ML_correction_dQu_dQv(model, T_mid, qv, cos_zenith, lat, phis, u, v, dt): + """Get ML correction for eastward wind (dQu or dQxwind) and northward wind (dQv or dQywind) + + Args: + model: pre-trained ML model for dQu and dQv + T_mid: air temperature + qv: specific humidity + cos_zenith: cosine zenith angle + lat: latitude + phis: surface geopotential + u: horizontal wind in x-direction + v: horizontal wind in y-direction + dt: time step (s) + """ + ds = xr.Dataset( + data_vars=dict( + T_mid=(["ncol", "z"], T_mid), + qv=(["ncol", "z"], qv), + U=(["ncol", "z"], u), + V=(["ncol", "z"], v), + lat=(["ncol"], lat), + surface_geopotential=(["ncol"], phis), + cos_zenith_angle=(["ncol"], cos_zenith), + ) + ) + output = ensure_correction_ordering(predict(model, ds, dt)) + # rename dQxwind and dQywind to dQu and dQv if needed + if "dQxwind" in output.keys(): + output["dQu"] = output.pop("dQxwind") + if "dQywind" in output.keys(): + output["dQv"] = output.pop("dQywind") + + return output + + +def get_ML_correction_sfc_fluxes( + model, + T_mid, + qv, + cos_zenith, + lat, + phis, + sfc_alb_dif_vis, + sw_flux_dn, + dt, +): + """Get ML correction for overriding surface fluxes (net shortwave and downward longwave) + ML model should have the following output variables: + net_shortwave_sfc_flux_via_transmissivity + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + + Args: + model: pre-trained ML model for radiative fluxes + T_mid: air temperature + qv: specific humidity + cos_zenith: cosine zenith angle + lat: latitude + phis: surface geopotential + sfc_alb_dif_vis: surface albedo for diffuse shortwave radiation + sw_flux_dn: downward shortwave flux + dt: time step (s) + """ + SW_flux_dn_at_model_top = sw_flux_dn[:, 0] + ds = xr.Dataset( + data_vars=dict( + T_mid=(["ncol", "z"], T_mid), + qv=(["ncol", "z"], qv), + lat=(["ncol"], lat), + surface_geopotential=(["ncol"], phis), + cos_zenith_angle=(["ncol"], cos_zenith), + surface_diffused_shortwave_albedo=(["ncol"], sfc_alb_dif_vis), + total_sky_downward_shortwave_flux_at_top_of_atmosphere=( + ["ncol"], + SW_flux_dn_at_model_top, + ), + ) + ) + return predict(model, ds, dt) + + +def update_fields( + T_mid, + qv, + u, + v, + lat, + lon, + phis, + sw_flux_dn, + sfc_alb_dif_vis, + sfc_flux_sw_net, + sfc_flux_lw_dn, + Ncol, + Nlev, + num_tracers, + dt, + model_tq, + model_uv, + model_sfc_fluxes, + current_time, +): + """ + T_mid: temperature + qv: specific humidity + u: x-component of wind + v: y-component of wind + lat: latitude + lon: longitude + phis: surface geopotential + SW_flux_dn_at_model_top: downwelling shortwave flux at the top of the model + sfc_alb_dif_vis: surface diffuse shortwave albedo + sfc_flux_sw_net + sfc_flux_lw_dn + Ncol: number of columns + Nlev: number of levels + num_tracers: number of tracers + dt: time step (s) + model_tq: path to the ML model for temperature and specific humidity + model_uv: path to the ML model for u and v + current_time: current time in the format "YYYY-MM-DD HH:MM:SS" + """ + T_mid = np.reshape(T_mid, (-1, Nlev)) + u = np.reshape(u, (-1, Nlev)) + v = np.reshape(v, (-1, Nlev)) + # qv is a 3D array of shape (Ncol, num_tracers, Nlev) + # by default, qv is the frist tracer variable + qv = np.reshape(qv, (-1, num_tracers, Nlev)) + current_datetime = datetime.datetime.strptime(current_time, "%Y-%m-%d %H:%M:%S") + cos_zenith = cos_zenith_angle( + current_datetime, + lon, + lat, + ) + if model_tq is not None: + correction_tq = get_ML_correction_dQ1_dQ2( + model_tq, T_mid, qv[:, 0, :], cos_zenith, dt + ) + T_mid[:, :] += correction_tq["dQ1"].values * dt + qv[:, 0, :] += correction_tq["dQ2"].values * dt + if model_uv is not None: + correction_uv = get_ML_correction_dQu_dQv( + model_uv, T_mid, qv[:, 0, :], cos_zenith, lat, phis, u, v, dt + ) + u[:, :] += correction_uv["dQu"].values * dt + v[:, :] += correction_uv["dQv"].values * dt + if model_sfc_fluxes is not None: + sw_flux_dn = np.reshape(sw_flux_dn, (-1, Nlev+1)) + correction_sfc_fluxes = get_ML_correction_sfc_fluxes( + model_sfc_fluxes, + T_mid, + qv[:, 0, :], + cos_zenith, + lat, + phis, + sfc_alb_dif_vis, + sw_flux_dn, + dt, + ) + sfc_flux_sw_net[:] = correction_sfc_fluxes["net_shortwave_sfc_flux_via_transmissivity"].values + sfc_flux_lw_dn[:] = correction_sfc_fluxes["override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface"].values diff --git a/components/eamxx/src/physics/nudging/CMakeLists.txt b/components/eamxx/src/physics/nudging/CMakeLists.txt index f75a24a6d02d..faf345d09012 100644 --- a/components/eamxx/src/physics/nudging/CMakeLists.txt +++ b/components/eamxx/src/physics/nudging/CMakeLists.txt @@ -1,5 +1,5 @@ -add_library(nudging atmosphere_nudging.cpp) -target_include_directories(nudging PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../share) +add_library(nudging eamxx_nudging_process_interface.cpp) +target_compile_definitions(nudging PUBLIC EAMXX_HAS_NUDGING) target_link_libraries(nudging physics_share scream_share) if (NOT SCREAM_LIB_ONLY) diff --git a/components/eamxx/src/physics/nudging/atmosphere_nudging.cpp b/components/eamxx/src/physics/nudging/atmosphere_nudging.cpp deleted file mode 100644 index 03fc2a5563c8..000000000000 --- a/components/eamxx/src/physics/nudging/atmosphere_nudging.cpp +++ /dev/null @@ -1,352 +0,0 @@ -#include "atmosphere_nudging.hpp" - -namespace scream -{ - - //using namespace spa; -// ========================================================================================= -Nudging::Nudging (const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereProcess(comm, params) -{ - datafile=m_params.get("Nudging_Filename"); -} - -// ========================================================================================= -void Nudging::set_grids(const std::shared_ptr grids_manager) -{ - using namespace ekat::units; - using namespace ShortFieldTagsNames; - - m_grid = grids_manager->get_grid("Physics"); - const auto& grid_name = m_grid->name(); - m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank - m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; - - constexpr int ps = 1; - auto Q = kg/kg; - Q.set_string("kg/kg"); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - add_field("u", scalar3d_layout_mid, m/s, grid_name, ps); - add_field("v", scalar3d_layout_mid, m/s, grid_name, ps); - - //Now need to read in the file - scorpio::register_file(datafile,scorpio::Read); - m_num_src_levs = scorpio::get_dimlen(datafile,"lev"); - double time_value_1= scorpio::read_time_at_index_c2f(datafile.c_str(),1); - double time_value_2= scorpio::read_time_at_index_c2f(datafile.c_str(),2); - - //Here we are assuming that the time in the netcdf file is in days - //Internally we want this in seconds so need to convert - //Only consider integer time steps in seconds to resolve any roundoff error - time_step_file=int((time_value_2-time_value_1)*86400); - scorpio::eam_pio_closefile(datafile); -} - -// ========================================================================================= -void Nudging::initialize_impl (const RunType /* run_type */) -{ - using namespace ShortFieldTagsNames; - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_src_levs} }; - FieldLayout horiz_wind_layout { {COL,CMP,LEV}, {m_num_cols,2,m_num_src_levs} }; - fields_ext["T_mid"] = view_2d("T_mid",m_num_cols,m_num_src_levs); - fields_ext_h["T_mid"] = Kokkos::create_mirror_view(fields_ext["T_mid"]); - auto T_mid_h=fields_ext_h["T_mid"]; - host_views["T_mid"] = view_1d_host(T_mid_h.data(),T_mid_h.size()); - layouts.emplace("T_mid", scalar3d_layout_mid); - - fields_ext["p_mid"] = view_2d("p_mid",m_num_cols,m_num_src_levs); - fields_ext_h["p_mid"] = Kokkos::create_mirror_view(fields_ext["p_mid"]); - auto p_mid_h=fields_ext_h["p_mid"]; - host_views["p_mid"] = view_1d_host(p_mid_h.data(),p_mid_h.size()); - layouts.emplace("p_mid", scalar3d_layout_mid); - - fields_ext["qv"] = view_2d("qv",m_num_cols,m_num_src_levs); - fields_ext_h["qv"] = Kokkos::create_mirror_view(fields_ext["qv"]); - auto qv_h=fields_ext_h["qv"]; - host_views["qv"] = view_1d_host(qv_h.data(),qv_h.size()); - layouts.emplace("qv", scalar3d_layout_mid); - - fields_ext["u"] = view_2d("u",m_num_cols,m_num_src_levs); - fields_ext_h["u"] = Kokkos::create_mirror_view(fields_ext["u"]); - auto u_h=fields_ext_h["u"]; - host_views["u"] = view_1d_host(u_h.data(),u_h.size()); - layouts.emplace("u", scalar3d_layout_mid); - - fields_ext["v"] = view_2d("v",m_num_cols,m_num_src_levs); - fields_ext_h["v"] = Kokkos::create_mirror_view(fields_ext["v"]); - auto v_h=fields_ext_h["v"]; - host_views["v"] = view_1d_host(v_h.data(),v_h.size()); - layouts.emplace("v", scalar3d_layout_mid); - - auto grid_l = m_grid->clone("Point Grid", false); - grid_l->reset_num_vertical_lev(m_num_src_levs); - ekat::ParameterList data_in_params; - m_fnames = {"T_mid","p_mid","qv","u","v"}; - data_in_params.set("Field Names",m_fnames); - data_in_params.set("Filename",datafile); - // We need to skip grid checks because multiple ranks - // may want the same column of source data. - data_in_params.set("Skip_Grid_Checks",true); - data_input.init(data_in_params,grid_l,host_views,layouts); - - T_mid_ext = fields_ext["T_mid"]; - p_mid_ext = fields_ext["p_mid"]; - qv_ext = fields_ext["qv"]; - u_ext = fields_ext["u"]; - v_ext = fields_ext["v"]; - ts0=timestamp(); - - //Check that internal timestamp starts at same point as time in external file - int start_date=scorpio::get_attribute(datafile,"start_date"); - int start_time=scorpio::get_attribute(datafile,"start_time"); - int start_year=int(start_date/10000); - int start_month=int((start_date-start_year*10000)/100); - int start_day=int(start_date-start_year*10000-start_month*100); - int start_hour=int(start_time/10000); - int start_min=int((start_time-start_hour*10000)/100); - int start_sec=int(start_time-start_hour*10000-start_min*100); - - EKAT_REQUIRE_MSG(start_year==ts0.get_year(), - "ERROR: The start year from the nudging file is "\ - "different than the internal simulation start year\n"); - EKAT_REQUIRE_MSG(start_month==ts0.get_month(), - "ERROR: The start month from the nudging file is "\ - "different than the internal simulation start month\n"); - EKAT_REQUIRE_MSG(start_day==ts0.get_day(), - "ERROR: The start day from the nudging file is "\ - "different than the internal simulation start day\n"); - EKAT_REQUIRE_MSG(start_hour==ts0.get_hours(), - "ERROR: The start hour from the nudging file is "\ - "different than the internal simulation start hour\n"); - EKAT_REQUIRE_MSG(start_min==ts0.get_minutes(), - "ERROR: The start minute from the nudging file is "\ - "different than the internal simulation start minute\n"); - EKAT_REQUIRE_MSG(start_sec==ts0.get_seconds(), - "ERROR: The start second from the nudging file is "\ - "different than the internal simulation start second\n"); - - //Initialize before and after data - NudgingData_bef.init(m_num_cols,m_num_src_levs,true); - NudgingData_bef.time = -999; - NudgingData_aft.init(m_num_cols,m_num_src_levs,true); - NudgingData_aft.time = -999; - - //Read in the first time step - data_input.read_variables(0); - Kokkos::deep_copy(NudgingData_bef.T_mid,fields_ext_h["T_mid"]); - Kokkos::deep_copy(NudgingData_bef.p_mid,fields_ext_h["p_mid"]); - Kokkos::deep_copy(NudgingData_bef.u,fields_ext_h["u"]); - Kokkos::deep_copy(NudgingData_bef.v,fields_ext_h["v"]); - Kokkos::deep_copy(NudgingData_bef.qv,fields_ext_h["qv"]); - NudgingData_bef.time = 0.; - - //Read in the first time step - data_input.read_variables(1); - Kokkos::deep_copy(NudgingData_aft.T_mid,fields_ext_h["T_mid"]); - Kokkos::deep_copy(NudgingData_aft.p_mid,fields_ext_h["p_mid"]); - Kokkos::deep_copy(NudgingData_aft.u,fields_ext_h["u"]); - Kokkos::deep_copy(NudgingData_aft.v,fields_ext_h["v"]); - Kokkos::deep_copy(NudgingData_aft.qv,fields_ext_h["qv"]); - NudgingData_aft.time = time_step_file; - -} - -void Nudging::time_interpolation (const int time_s) { - - using KT = KokkosTypes; - using ExeSpace = typename KT::ExeSpace; - using ESU = ekat::ExeSpaceUtils; - using MemberType = typename KT::MemberType; - - const int time_index = time_s/time_step_file; - double time_step_file_d = time_step_file; - double w_bef = ((time_index+1)*time_step_file-time_s) / time_step_file_d; - double w_aft = (time_s-(time_index)*time_step_file) / time_step_file_d; - const int num_cols = NudgingData_aft.T_mid.extent(0); - const int num_vert_packs = NudgingData_aft.T_mid.extent(1); - const auto policy = ESU::get_default_team_policy(num_cols, num_vert_packs); - - auto T_mid_bef = NudgingData_bef.T_mid; - auto T_mid_aft = NudgingData_aft.T_mid; - auto u_bef = NudgingData_bef.u; - auto u_aft = NudgingData_aft.u; - auto v_bef = NudgingData_bef.v; - auto v_aft = NudgingData_aft.v; - auto p_mid_bef = NudgingData_bef.p_mid; - auto p_mid_aft = NudgingData_aft.p_mid; - auto qv_bef = NudgingData_bef.qv; - auto qv_aft = NudgingData_aft.qv; - - Kokkos::parallel_for("nudging_time_interpolation", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank(); - auto T_mid_1d = ekat::subview(T_mid_ext,icol); - auto T_mid_bef_1d = ekat::subview(T_mid_bef,icol); - auto T_mid_aft_1d = ekat::subview(T_mid_aft,icol); - auto u_1d = ekat::subview(u_ext,icol); - auto u_bef_1d = ekat::subview(u_bef,icol); - auto u_aft_1d = ekat::subview(u_aft,icol); - auto v_1d = ekat::subview(v_ext,icol); - auto v_bef_1d = ekat::subview(v_bef,icol); - auto v_aft_1d = ekat::subview(v_aft,icol); - auto p_mid_1d = ekat::subview(p_mid_ext,icol); - auto p_mid_bef_1d = ekat::subview(p_mid_bef,icol); - auto p_mid_aft_1d = ekat::subview(p_mid_aft,icol); - auto qv_1d = ekat::subview(qv_ext,icol); - auto qv_bef_1d = ekat::subview(qv_bef,icol); - auto qv_aft_1d = ekat::subview(qv_aft,icol); - - const auto range = Kokkos::TeamThreadRange(team, num_vert_packs); - Kokkos::parallel_for(range, [&] (const Int & k) { - T_mid_1d(k)=w_bef*T_mid_bef_1d(k) + w_aft*T_mid_aft_1d(k); - p_mid_1d(k)=w_bef*p_mid_bef_1d(k) + w_aft*p_mid_aft_1d(k); - u_1d(k)=w_bef*u_bef_1d(k) + w_aft*u_aft_1d(k); - v_1d(k)=w_bef*v_bef_1d(k) + w_aft*v_aft_1d(k); - qv_1d(k)=w_bef*qv_bef_1d(k) + w_aft*qv_aft_1d(k); - }); - team.team_barrier(); - }); - Kokkos::fence(); -} - -// ========================================================================================= -void Nudging::update_time_step (const int time_s) -{ - //Check to see in time state needs to be updated - if (time_s >= NudgingData_aft.time) - { - const int time_index = time_s/time_step_file; - std::swap (NudgingData_bef,NudgingData_aft); - NudgingData_bef.time = NudgingData_aft.time; - - data_input.read_variables(time_index+1); - Kokkos::deep_copy(NudgingData_aft.T_mid,fields_ext_h["T_mid"]); - Kokkos::deep_copy(NudgingData_aft.p_mid,fields_ext_h["p_mid"]); - Kokkos::deep_copy(NudgingData_aft.u,fields_ext_h["u"]); - Kokkos::deep_copy(NudgingData_aft.v,fields_ext_h["v"]); - Kokkos::deep_copy(NudgingData_aft.qv,fields_ext_h["qv"]); - NudgingData_aft.time = time_step_file*(time_index+1); - } - -} - - -// ========================================================================================= -void Nudging::run_impl (const double dt) -{ - using namespace scream::vinterp; - - //Have to add dt because first time iteration is at 0 seconds where you will - //not have any data from the field. The timestamp is only iterated at the - //end of the full step in scream. - auto ts = timestamp()+dt; - auto time_since_zero = ts.seconds_from(ts0); - - //update time state information and check whether need to update data - update_time_step(time_since_zero); - - //perform time interpolation - time_interpolation(time_since_zero); - - auto T_mid = get_field_out("T_mid").get_view(); - auto qv = get_field_out("qv").get_view(); - auto u = get_field_out("u").get_view(); - auto v = get_field_out("v").get_view(); - - const auto& p_mid = get_field_in("p_mid").get_view(); - - const view_Nd T_mid_ext_p(reinterpret_cast(fields_ext["T_mid"].data()), - m_num_cols,m_num_src_levs); - const view_Nd p_mid_ext_p(reinterpret_cast(fields_ext["p_mid"].data()), - m_num_cols,m_num_src_levs); - const view_Nd qv_ext_p(reinterpret_cast(fields_ext["qv"].data()), - m_num_cols,m_num_src_levs); - const view_Nd u_ext_p(reinterpret_cast(fields_ext["u"].data()), - m_num_cols,m_num_src_levs); - const view_Nd v_ext_p(reinterpret_cast(fields_ext["v"].data()), - m_num_cols,m_num_src_levs); - - //Now perform the vertical interpolation from the external file to the internal - //field levels - perform_vertical_interpolation(p_mid_ext_p, - p_mid, - T_mid_ext_p, - T_mid, - m_num_src_levs, - m_num_levs); - - perform_vertical_interpolation(p_mid_ext_p, - p_mid, - qv_ext_p, - qv, - m_num_src_levs, - m_num_levs); - - perform_vertical_interpolation(p_mid_ext_p, - p_mid, - u_ext_p, - u, - m_num_src_levs, - m_num_levs); - - perform_vertical_interpolation(p_mid_ext_p, - p_mid, - v_ext_p, - v, - m_num_src_levs, - m_num_levs); - - //This code removes any masked values and sets them to the corresponding - //values at the lowest/highest pressure levels from the external file - const int num_cols = T_mid.extent(0); - const int num_vert_packs = T_mid.extent(1); - const int num_vert_packs_ext = T_mid_ext.extent(1); - const auto policy = ESU::get_default_team_policy(num_cols, num_vert_packs); - Kokkos::parallel_for("correct_for_masked_values", policy, - KOKKOS_LAMBDA(MemberType const& team) { - const int icol = team.league_rank(); - auto T_mid_1d = ekat::subview(T_mid,icol); - auto T_mid_ext_1d = ekat::subview(T_mid_ext,icol); - auto u_1d = ekat::subview(u,icol); - auto u_ext_1d = ekat::subview(u_ext,icol); - auto v_1d = ekat::subview(v,icol); - auto v_ext_1d = ekat::subview(v_ext,icol); - - auto p_mid_1d = ekat::subview(p_mid,icol); - auto p_mid_ext_1d = ekat::subview(p_mid_ext,icol); - auto qv_1d = ekat::subview(qv,icol); - auto qv_ext_1d = ekat::subview(qv_ext,icol); - const auto range = Kokkos::TeamThreadRange(team, num_vert_packs); - Kokkos::parallel_for(range, [&] (const Int & k) { - const auto above_max = p_mid_1d(k) > p_mid_ext_1d(num_vert_packs_ext-1); - const auto below_min = p_mid_1d(k) < p_mid_ext_1d(0); - if (above_max.any()){ - T_mid_1d(k).set(above_max,T_mid_ext_1d(num_vert_packs_ext-1)); - qv_1d(k).set(above_max,qv_ext_1d(num_vert_packs_ext-1)); - u_1d(k).set(above_max,u_ext_1d(num_vert_packs_ext-1)); - v_1d(k).set(above_max,v_ext_1d(num_vert_packs_ext-1)); - } - if (below_min.any()){ - T_mid_1d(k).set(below_min,T_mid_ext_1d(0)); - qv_1d(k).set(below_min,qv_ext_1d(0)); - u_1d(k).set(below_min,u_ext_1d(0)); - v_1d(k).set(below_min,v_ext_1d(0)); - } - }); - team.team_barrier(); - }); - Kokkos::fence(); -} - -// ========================================================================================= -void Nudging::finalize_impl() -{ - data_input.finalize(); -} - -} // namespace scream diff --git a/components/eamxx/src/physics/nudging/atmosphere_nudging.hpp b/components/eamxx/src/physics/nudging/atmosphere_nudging.hpp deleted file mode 100644 index 9414758f2696..000000000000 --- a/components/eamxx/src/physics/nudging/atmosphere_nudging.hpp +++ /dev/null @@ -1,104 +0,0 @@ -#ifndef SCREAM_NUDGING_HPP -#define SCREAM_NUDGING_HPP - -#include "share/atm_process/atmosphere_process.hpp" -#include "ekat/ekat_parameter_list.hpp" -#include "ekat/util/ekat_lin_interp.hpp" -#include "share/io/scream_output_manager.hpp" -#include "share/io/scorpio_output.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_scorpio_interface.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/grid/point_grid.hpp" -#include "share/util/scream_vertical_interpolation.hpp" -#include "share/util/scream_time_stamp.hpp" -#include "physics/nudging/nudging_functions.hpp" - -#include - -namespace scream -{ - -/* - * The class responsible to handle the nudging of variables -*/ - -class Nudging : public AtmosphereProcess -{ -public: - using NudgingFunc = nudging::NudgingFunctions; - using mPack = ekat::Pack; - using KT = KokkosTypes; - - template - using view_1d = typename KT::template view_1d; - - template - using view_2d = typename KT::template view_2d; - - template - using view_Nd_host = typename KT::template view_ND::HostMirror; - - template - using view_1d_host = view_Nd_host; - - template - using view_2d_host = view_Nd_host; - - // Constructors - Nudging (const ekat::Comm& comm, const ekat::ParameterList& params); - - // The type of subcomponent - AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } - - // The name of the subcomponent - std::string name () const { return "Nudging"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - - //Update the time step - void update_time_step(const int time_s); - - //Time interpolation function - void time_interpolation(const int time_s); - -#ifndef KOKKOS_ENABLE_CUDA - // Cuda requires methods enclosing __device__ lambda's to be public -protected: -#endif - - void run_impl (const double dt); - -protected: - - // The three main overrides for the subcomponent - void initialize_impl (const RunType run_type); - void finalize_impl (); - - std::shared_ptr m_grid; - // Keep track of field dimensions and the iteration count - int m_num_cols; - int m_num_levs; - int m_num_src_levs; - int time_step_file; - std::string datafile; - std::map> host_views; - std::map layouts; - std::vector m_fnames; - std::map> fields_ext; - std::map> fields_ext_h; - view_2d T_mid_ext; - view_2d p_mid_ext; - view_2d qv_ext; - view_2d u_ext; - view_2d v_ext; - TimeStamp ts0; - NudgingFunc::NudgingData NudgingData_bef; - NudgingFunc::NudgingData NudgingData_aft; - AtmosphereInput data_input; -}; // class Nudging - -} // namespace scream - -#endif // SCREAM_NUDGING_HPP diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp new file mode 100644 index 000000000000..1dff45c7d032 --- /dev/null +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -0,0 +1,603 @@ +#include "eamxx_nudging_process_interface.hpp" +#include "share/util/scream_universal_constants.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/grid/remap/do_nothing_remapper.hpp" + +namespace scream +{ + +// ========================================================================================= +Nudging::Nudging (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereProcess(comm, params) +{ + m_datafiles = m_params.get>("nudging_filename"); + m_timescale = m_params.get("nudging_timescale",0); + m_fields_nudge = m_params.get>("nudging_fields"); + m_use_weights = m_params.get("use_nudging_weights",false); + // If we are doing horizontal refine-remapping, we need to get the mapfile from user + m_refine_remap_file = m_params.get( + "nudging_refine_remap_mapfile", "no-file-given"); + m_refine_remap_vert_cutoff = m_params.get( + "nudging_refine_remap_vert_cutoff", 0.0); + auto src_pres_type = m_params.get("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + if (src_pres_type=="TIME_DEPENDENT_3D_PROFILE") { + m_src_pres_type = TIME_DEPENDENT_3D_PROFILE; + } else if (src_pres_type=="STATIC_1D_VERTICAL_PROFILE") { + m_src_pres_type = STATIC_1D_VERTICAL_PROFILE; + // Check for a designated source pressure file, default to first nudging data source if not given. + m_static_vertical_pressure_file = m_params.get("source_pressure_file",m_datafiles[0]); + } else { + EKAT_ERROR_MSG("ERROR! Nudging::parameter_list - unsupported source_pressure_type provided. Current options are [TIME_DEPENDENT_3D_PROFILE,STATIC_1D_VERTICAL_PROFILE]. Please check"); + } + // use nudging weights + if (m_use_weights) + m_weights_file = m_params.get("nudging_weights_file"); + + // TODO: Add some warning messages here. + // 1. if m_timescale is <= 0 we will do direct replacement. + // 2. if m_fields_nudge is empty or =NONE then we will skip nudging altogether. +} + +// ========================================================================================= +void Nudging::set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + m_grid = grids_manager->get_grid("Physics"); + const auto& grid_name = m_grid->name(); + m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column + + FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; + FieldLayout horiz_wind_layout { {COL,CMP,LEV}, {m_num_cols,2,m_num_levs} }; + + constexpr int ps = 1; + auto Q = kg/kg; + Q.set_string("kg/kg"); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); + + /* ----------------------- WARNING --------------------------------*/ + /* The following is a HACK to get things moving, we don't want to + * add all fields as "updated" long-term. A separate stream of work + * is adapting the infrastructure to allow for a generic "add_field" call + * to be used here which we can then setup using the m_fields_nudge variable + * For now we check if a field is intended to be nudged via the m_fields_nudge + * vector, if it is we register it. For now we are limited to just T_mid, qv, + * U and V + */ + if (ekat::contains(m_fields_nudge,"T_mid")) { + add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); + } + if (ekat::contains(m_fields_nudge,"qv")) { + add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); + } + if (ekat::contains(m_fields_nudge,"U") or ekat::contains(m_fields_nudge,"V")) { + add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + } + + /* ----------------------- WARNING --------------------------------*/ + + //Now need to read in the file + if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { + m_num_src_levs = scorpio::get_dimlen(m_datafiles[0],"lev"); + } else { + m_num_src_levs = scorpio::get_dimlen(m_static_vertical_pressure_file,"lev"); + } + + /* Check for consistency between nudging files, map file, and remapper */ + + // Number of columns globally + auto m_num_cols_global = m_grid->get_num_global_dofs(); + + // Get the information from the first nudging data file + int num_cols_src = scorpio::get_dimlen(m_datafiles[0],"ncol"); + + if (num_cols_src != m_num_cols_global) { + // If differing cols, check if remap file is provided + EKAT_REQUIRE_MSG(m_refine_remap_file != "no-file-given", + "Error! Nudging::set_grids - the number of columns in the nudging data file " + << std::to_string(num_cols_src) << " does not match the number of columns in the " + << "model grid " << std::to_string(m_num_cols_global) << ". Please check the " + << "nudging data file and/or the model grid."); + // If remap file is provided, check if it is consistent with the nudging data file + // First get the data from the mapfile + int num_cols_remap_a = scorpio::get_dimlen(m_refine_remap_file,"n_a"); + int num_cols_remap_b = scorpio::get_dimlen(m_refine_remap_file,"n_b"); + // Then, check if n_a (source) and n_b (target) are consistent + EKAT_REQUIRE_MSG(num_cols_remap_a == num_cols_src, + "Error! Nudging::set_grids - the number of columns in the nudging data file " + << std::to_string(num_cols_src) << " does not match the number of columns in the " + << "mapfile " << std::to_string(num_cols_remap_a) << ". Please check the " + << "nudging data file and/or the mapfile."); + EKAT_REQUIRE_MSG(num_cols_remap_b == m_num_cols_global, + "Error! Nudging::set_grids - the number of columns in the model grid " + << std::to_string(m_num_cols_global) << " does not match the number of columns in the " + << "mapfile " << std::to_string(num_cols_remap_b) << ". Please check the " + << "model grid and/or the mapfile."); + EKAT_REQUIRE_MSG(m_use_weights == false, + "Error! Nudging::set_grids - it seems that the user intends to use both nuding " + << "from coarse data as well as weighted nudging simultaneously. This is not supported. " + << "If the user wants to use both at their own risk, the user should edit the source code " + << "by deleting this error message."); + // If we get here, we are good to go! + m_refine_remap = true; + } else { + // If the number of columns is the same, we don't need to do any remapping, + // but print a warning if the user provided a mapfile + if (m_refine_remap_file != "no-file-given") { + std::cout << "Warning! Nudging::set_grids - the number of columns in the nudging data file " + << std::to_string(num_cols_src) << " matches the number of columns in the " + << "model grid " << std::to_string(m_num_cols_global) << ". The mapfile " + << m_refine_remap_file << " will NOT be used. Please check the " + << "nudging data file and/or the model grid." << std::endl; + } + // If the user gives us the vertical cutoff, warn them + if (m_refine_remap_vert_cutoff > 0.0) { + std::cout << "Warning! Nudging::set_grids - the vertical cutoff " + << std::to_string(m_refine_remap_vert_cutoff) + << " is larger than zero, but we are not remapping. Please " + "check your settings." << std::endl; + } + // Set m_refine_remap to false + m_refine_remap = false; + } +} +// ========================================================================================= +void Nudging::apply_tendency(Field& base, const Field& next, const Real dt) +{ + // Calculate the weight to apply the tendency + const Real dtend = dt/Real(m_timescale); + EKAT_REQUIRE_MSG(dtend>=0,"Error! Nudging::apply_tendency - timescale tendency of " << std::to_string(dt) + << " / " << std::to_string(m_timescale) << " = " << std::to_string(dtend) + << " is invalid. Please check the timescale and/or dt"); + // Now apply the tendency. + Field tend = base.clone(); + // Use update internal to set tendency, will be (1.0*next - 1.0*base), note tend=base at this point. + tend.update(next,Real(1.0),Real(-1.0)); + base.update(tend,dtend,Real(1.0)); +} +// ========================================================================================= +void Nudging::apply_weighted_tendency(Field& base, const Field& next, const Field& weights, const Real dt) +{ + // Calculate the weight to apply the tendency + const Real dtend = dt/Real(m_timescale); + EKAT_REQUIRE_MSG(dtend>=0,"Error! Nudging::apply_tendency - timescale tendency of " << std::to_string(dt) + << " / " << std::to_string(m_timescale) << " = " << std::to_string(dtend) + << " is invalid. Please check the timescale and/or dt"); + // Now apply the tendency. + Field tend = base.clone(); + + // Use update internal to set tendency, will be (weights*next - weights*base), note tend=base at this point. + auto base_view = base.get_view(); + auto tend_view = tend.get_view< Real**>(); + auto next_view = next.get_view< Real**>(); + auto w_view = weights.get_view< Real**>(); + + const int num_cols = base_view.extent(0); + const int num_vert_packs = base_view.extent(1); + Kokkos::parallel_for(Kokkos::MDRangePolicy>({0, 0}, {num_cols, num_vert_packs}), KOKKOS_LAMBDA(int i, int j) { + tend_view(i,j) = next_view(i,j)*w_view(i,j) - base_view(i,j)*w_view(i,j); + }); + base.update(tend, dtend, Real(1.0)); +} +// ========================================================================================= +void Nudging::apply_vert_cutoff_tendency(Field &base, const Field &next, + const Field &p_mid, const Real cutoff, + const Real dt) { + // Calculate the weight to apply the tendency + const Real dtend = dt / Real(m_timescale); + EKAT_REQUIRE_MSG(dtend >= 0, + "Error! Nudging::apply_tendency - timescale tendency of " + << std::to_string(dt) << " / " + << std::to_string(m_timescale) << " = " + << std::to_string(dtend) + << " is invalid. Please check the timescale and/or dt"); + // Now apply the tendency. + Field tend = base.clone(); + + // Use update internal to set tendency, will be (weights*next - weights*base), + // note tend=base at this point. + auto base_view = base.get_view(); + auto tend_view = tend.get_view(); + auto next_view = next.get_view(); + auto pmid_view = p_mid.get_view(); + + const int num_cols = base_view.extent(0); + const int num_vert_packs = base_view.extent(1); + Kokkos::parallel_for( + Kokkos::MDRangePolicy>({0, 0}, + {num_cols, num_vert_packs}), + KOKKOS_LAMBDA(int i, int j) { + // If the pressure is above the cutoff, then we are closer to the surface + if (pmid_view(i, j) > cutoff) { + // Don't apply the tendency closer to the surface + tend_view(i, j) = 0.0; + } else { + // Apply the tendency farther up from the surface + tend_view(i, j) = next_view(i, j) - base_view(i, j); + } + }); + base.update(tend, dtend, Real(1.0)); +} +// ============================================================================================================= +void Nudging::initialize_impl (const RunType /* run_type */) +{ + using namespace ShortFieldTagsNames; + // Set up pointers for grids + // external grid: from source data + std::shared_ptr grid_ext; + // temporary grid: after vertical interpolation + std::shared_ptr grid_tmp; + // internal grid: after horizontal interpolation + std::shared_ptr grid_int; + + // Initialize the refining remapper stuff at the outset, + // because we need to know the grid information; + // for now, we are doing the horizontal interpolation last, + // so we use the m_grid (model physics) as the target grid + grid_int = m_grid->clone(m_grid->name(), true); + if (m_refine_remap) { + // P2P remapper + m_refine_remapper = + std::make_shared(grid_int, m_refine_remap_file); + // Get grid from remapper, and clone it + auto grid_ext_const = m_refine_remapper->get_src_grid(); + grid_ext = grid_ext_const->clone(grid_ext_const->name(), true); + // Finally, grid_ext may have different levels + grid_ext->reset_num_vertical_lev(m_num_src_levs); + } else { + // We set up a DoNothingRemapper, which will do nothing + m_refine_remapper = std::make_shared(grid_int, grid_int); + // We clone physics grid, but maybe we have different levels + grid_ext = m_grid->clone(m_grid->name(), true); + grid_ext->reset_num_vertical_lev(m_num_src_levs); + } + // The temporary grid is the external grid, but with + // the same number of levels as the internal (physics) grid + grid_tmp = grid_ext->clone(grid_ext->name(), true); + grid_tmp->reset_num_vertical_lev(m_num_levs); + + // Declare the layouts for the helper fields (int: internal) + FieldLayout scalar2d_layout_mid { {LEV}, {m_num_levs} }; + FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; + + // Get the number of external cols on current rank + auto m_num_cols_ext = grid_ext->get_num_local_dofs(); + // Declare the layouts for the helper fields (tmp: temporary)) + FieldLayout scalar2d_layout_mid_tmp { {LEV}, {m_num_levs}}; + FieldLayout scalar3d_layout_mid_tmp { {COL,LEV}, {m_num_cols_ext, m_num_levs} }; + // Declare the layouts for the helper fields (ext: external) + FieldLayout scalar2d_layout_mid_ext { {LEV}, {m_num_src_levs}}; + FieldLayout scalar3d_layout_mid_ext { {COL,LEV}, {m_num_cols_ext, m_num_src_levs} }; + + // Initialize the time interpolator + m_time_interp = util::TimeInterpolation(grid_ext, m_datafiles); + + constexpr int ps = SCREAM_PACK_SIZE; + // To be extra careful, this should be the ext_grid + const auto& grid_ext_name = grid_ext->name(); + if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { + auto pmid_ext = create_helper_field("p_mid_ext", scalar3d_layout_mid_ext, grid_ext_name, ps); + m_time_interp.add_field(pmid_ext.alias("p_mid"),true); + } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { + // Load p_levs from source data file + ekat::ParameterList in_params; + in_params.set("Filename",m_static_vertical_pressure_file); + in_params.set("Skip_Grid_Checks",true); // We need to skip grid checks because multiple ranks may want the same column of source data. + std::map> host_views; + std::map layouts; + auto pmid_ext = create_helper_field("p_mid_ext", scalar2d_layout_mid_ext, grid_ext_name, ps); + auto pmid_ext_v = pmid_ext.get_view(); + in_params.set>("Field Names",{"p_levs"}); + host_views["p_levs"] = pmid_ext_v; + layouts.emplace("p_levs",scalar2d_layout_mid_ext); + AtmosphereInput src_input(in_params,grid_ext,host_views,layouts); + src_input.read_variables(-1); + src_input.finalize(); + pmid_ext.sync_to_dev(); + } + + // Open the registration! + m_refine_remapper->registration_begins(); + + // To create helper fields for later; we do both tmp and ext... + for (auto name : m_fields_nudge) { + std::string name_ext = name + "_ext"; + std::string name_tmp = name + "_tmp"; + // Helper fields that will temporarily store the target state, which can then + // be used to back out a nudging tendency + auto grid_int_name = grid_int->name(); + auto grid_ext_name = grid_ext->name(); + auto grid_tmp_name = grid_tmp->name(); + auto field = get_field_out_wrap(name); + auto layout = field.get_header().get_identifier().get_layout(); + auto field_ext = create_helper_field(name_ext, scalar3d_layout_mid_ext, grid_ext_name, ps); + auto field_tmp = create_helper_field(name_tmp, scalar3d_layout_mid_tmp, grid_tmp_name, ps); + Field field_int; + if (m_refine_remap) { + field_int = create_helper_field(name, scalar3d_layout_mid, grid_int_name, ps); + } else { + field_int = field_tmp.alias(name); + m_helper_fields[name] = field_int; + } + + // Register the fields with the remapper + m_refine_remapper->register_field(field_tmp, field_int); + // Add the fields to the time interpolator + m_time_interp.add_field(field_ext.alias(name), true); + } + m_time_interp.initialize_data_from_files(); + + // Close the registration! + m_refine_remapper->registration_ends(); + + // load nudging weights from file + // NOTE: the regional nudging use the same grid as the run, no need to + // do the interpolation. + if (m_use_weights) + { + auto grid_name = m_grid->name(); + auto nudging_weights = create_helper_field("nudging_weights", scalar3d_layout_mid, grid_name, ps); + std::vector fields; + fields.push_back(nudging_weights); + AtmosphereInput src_weights_input(m_weights_file, m_grid, fields); + src_weights_input.read_variables(); + src_weights_input.finalize(); + nudging_weights.sync_to_dev(); + } +} + +// ========================================================================================= +void Nudging::run_impl (const double dt) +{ + using namespace scream::vinterp; + + // Have to add dt because first time iteration is at 0 seconds where you will + // not have any data from the field. The timestamp is only iterated at the + // end of the full step in scream. + auto ts = timestamp()+dt; + + // Perform time interpolation + m_time_interp.perform_time_interpolation(ts); + + // Process data and nudge the atmosphere state + const auto& p_mid_v = get_field_in("p_mid").get_view(); + view_Nd p_mid_ext_p; + view_Nd p_mid_ext_1d; + if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { + p_mid_ext_p = get_helper_field("p_mid_ext").get_view(); + } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { + p_mid_ext_1d = get_helper_field("p_mid_ext").get_view(); + } + + for (auto name : m_fields_nudge) { + auto atm_state_field = get_field_out_wrap(name); // int horiz, int vert + auto int_state_field = get_helper_field(name); // int horiz, int vert + auto ext_state_field = get_helper_field(name+"_ext"); // ext horiz, ext vert + auto tmp_state_field = get_helper_field(name+"_tmp"); // ext horiz, int vert + auto ext_state_view = ext_state_field.get_view(); + auto tmp_state_view = tmp_state_field.get_view(); + auto atm_state_view = atm_state_field.get_view(); // TODO: Right now assume whatever field is defined on COLxLEV + auto int_state_view = int_state_field.get_view(); + auto int_mask_view = m_buffer.int_mask_view; + // Masked values in the source data can lead to strange behavior in the vertical interpolation. + // We pre-process the data and map any masked values (sometimes called "filled" values) to the + // nearest un-masked value. + // Here we are updating the ext_state_view, which is the time interpolated values taken from the nudging + // data. + Real var_fill_value = constants::DefaultFillValue().value; + // Query the helper field for the fill value, if not present use default + if (ext_state_field.get_header().has_extra_data("mask_value")) { + var_fill_value = ext_state_field.get_header().get_extra_data("mask_value"); + } + const int num_cols = ext_state_view.extent(0); + const int num_vert_packs = ext_state_view.extent(1); + const int num_src_levs = m_num_src_levs; + const auto policy = ESU::get_default_team_policy(num_cols, num_vert_packs); + Kokkos::parallel_for("correct_for_masked_values", policy, + KOKKOS_LAMBDA(MemberType const& team) { + const int icol = team.league_rank(); + auto ext_state_view_1d = ekat::subview(ext_state_view,icol); + Real fill_value; + int fill_idx = -1; + // Scan top to surf and backfill all values near TOM that are masked. + for (int kk=0; kk(p_mid_ext_p, + p_mid_v, + ext_state_view, + tmp_state_view, + int_mask_view, + m_num_src_levs, + m_num_levs); + } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { + perform_vertical_interpolation(p_mid_ext_1d, + p_mid_v, + ext_state_view, + tmp_state_view, + int_mask_view, + m_num_src_levs, + m_num_levs); + } + + // Check that none of the nudging targets are masked, if they are, set value to + // nearest unmasked value above. + // NOTE: We use an algorithm whichs scans from TOM to the surface. + // If TOM is masked we keep scanning until we hit an unmasked value, + // we then set all masked values above to the unmasked value. + // We continue scanning towards the surface until we hit an unmasked value, we + // then assign that masked value the most recent unmasked value, until we hit the + // surface. + // Here we change the int_state_view which represents the vertically interpolated fields onto + // the simulation grid. + const int num_levs = m_num_levs; + Kokkos::parallel_for("correct_for_masked_values", policy, + KOKKOS_LAMBDA(MemberType const& team) { + const int icol = team.league_rank(); + auto int_mask_view_1d = ekat::subview(int_mask_view,icol); + auto tmp_state_view_1d = ekat::subview(tmp_state_view,icol); + Real fill_value; + int fill_idx = -1; + // Scan top to surf and backfill all values near TOM that are masked. + for (int kk=0; kkremap(true); + + for (auto name : m_fields_nudge) { + auto atm_state_field = get_field_out_wrap(name); // int horiz, int vert + auto int_state_field = get_helper_field(name); // int horiz, int vert + auto atm_state_view = atm_state_field.get_view(); // TODO: Right now assume whatever field is defined on COLxLEV + auto int_state_view = int_state_field.get_view(); + // Apply the nudging tendencies to the ATM state + if (m_timescale <= 0) { + // We do direct replacement + Kokkos::deep_copy(atm_state_view,int_state_view); + } else { + // Back out a tendency and apply it. + if (m_use_weights) { + // get nudging weights field + // NOTES: do we really need the vertical interpolation for nudging weights? Since we are going to + // use the same grids as the case by providing the nudging weights file. + // I would not apply the vertical interpolation here, but it depends... + // + auto nudging_weights_field = get_helper_field("nudging_weights"); + // appply the nudging tendencies to the ATM states + apply_weighted_tendency(atm_state_field, int_state_field, nudging_weights_field, dt); + } else if (m_refine_remap_vert_cutoff > 0.0) { + // If we have a cutoff, we apply the tendency with p_mid cutoff + // First, get p_mid the field in the atm (i.e., int) state + auto p_mid_field = get_field_in("p_mid"); + // Then, call the tendency with a Heaviside-like cutoff + apply_vert_cutoff_tendency(atm_state_field, int_state_field, + p_mid_field, m_refine_remap_vert_cutoff, dt); + } else { + apply_tendency(atm_state_field, int_state_field, dt); + } + } + } +} + +// ========================================================================================= +void Nudging::finalize_impl() +{ + m_time_interp.finalize(); +} +// ========================================================================================= +Field Nudging::create_helper_field (const std::string& name, + const FieldLayout& layout, + const std::string& grid_name, + const int ps) +{ + using namespace ekat::units; + // For helper fields we don't bother w/ units, so we set them to non-dimensional + FieldIdentifier id(name,layout,Units::nondimensional(),grid_name); + + // Create the field. Init with NaN's, so we spot instances of uninited memory usage + Field f(id); + if (ps>=0) { + f.get_header().get_alloc_properties().request_allocation(ps); + } else { + f.get_header().get_alloc_properties().request_allocation(); + } + f.allocate_view(); + f.deep_copy(ekat::ScalarTraits::invalid()); + + m_helper_fields[name] = f; + return m_helper_fields[name]; +} + +// ========================================================================================= +size_t Nudging::requested_buffer_size_in_bytes() const { + return m_buffer.num_2d_midpoint_mask_views*m_num_cols*m_num_levs*sizeof(mMask); +} + +// ========================================================================================= +void Nudging::init_buffers(const ATMBufferManager& buffer_manager) { + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error, Nudging::init_buffers! Buffers size not sufficient.\n"); + mMask* mem = reinterpret_cast(buffer_manager.get_memory()); + + m_buffer.int_mask_view = decltype(m_buffer.int_mask_view)(mem,m_num_cols,m_num_levs); + mem += m_buffer.int_mask_view.size(); + + size_t used_mem = (reinterpret_cast(mem) - buffer_manager.get_memory())*sizeof(Real); + EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error: Nudging::init_buffers! Used memory != requested memory."); +} +// ========================================================================================= +Field Nudging::get_field_out_wrap(const std::string& field_name) { + if (field_name == "U" or field_name == "V") { + auto hw = get_field_out("horiz_winds"); + if (field_name == "U") { + return hw.get_component(0); + } else { + return hw.get_component(1); + } + } else { + return get_field_out(field_name); + } +} + +} // namespace scream diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp new file mode 100644 index 000000000000..89d05728bcd8 --- /dev/null +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp @@ -0,0 +1,160 @@ +#ifndef SCREAM_NUDGING_HPP +#define SCREAM_NUDGING_HPP + +#include "share/util/eamxx_time_interpolation.hpp" +#include "share/atm_process/atmosphere_process.hpp" +#include "ekat/ekat_parameter_list.hpp" +#include "ekat/util/ekat_lin_interp.hpp" +#include "share/io/scream_output_manager.hpp" +#include "share/io/scorpio_output.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/grid/point_grid.hpp" +#include "share/util/scream_vertical_interpolation.hpp" +#include "share/util/scream_time_stamp.hpp" +#include "share/grid/remap/abstract_remapper.hpp" + +#include + +namespace scream +{ + +/* + * The class responsible to handle the nudging of variables +*/ + +// enum to track how the source pressure levels are defined +enum SourcePresType { + TIME_DEPENDENT_3D_PROFILE = 0, // DEFAULT - source data should include time/spatially varying p_mid with dimensions (time, col, lev) + STATIC_1D_VERTICAL_PROFILE = 1, // source data includes p_levs which is a static set of levels in both space and time, with dimensions (lev) +}; + +class Nudging : public AtmosphereProcess +{ +public: + using mPack = ekat::Pack; + using mMask = ekat::Mask<1>; + using KT = KokkosTypes; + + template + using view_1d = typename KT::template view_1d; + + template + using view_2d = typename KT::template view_2d; + + using uview_2d_mask = Unmanaged>; + + template + using view_Nd_host = typename KT::template view_ND::HostMirror; + + template + using view_1d_host = view_Nd_host; + + template + using view_2d_host = view_Nd_host; + + // Constructors + Nudging (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The type of subcomponent + AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name () const { return "Nudging"; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + + // Internal function to apply nudging at specific timescale with weights + void apply_weighted_tendency(Field& base, const Field& next, const Field& weights, const Real dt); + + // Structure for storing local variables initialized using the ATMBufferManager + struct Buffer { + // 2D view + uview_2d_mask int_mask_view; + + // Total number of 2d views + static constexpr int num_2d_midpoint_mask_views = 1; + }; + +#ifndef KOKKOS_ENABLE_CUDA + // Cuda requires methods enclosing __device__ lambda's to be public +protected: +#endif + + void run_impl (const double dt); + + /* Nudge from coarse data */ + // See more details later in this file + // Must add this here to make it public for CUDA + // (refining) remapper vertically-weighted tendency application + void apply_vert_cutoff_tendency(Field &base, const Field &next, + const Field &p_mid, const Real cutoff, + const Real dt); +protected: + + Field get_field_out_wrap(const std::string& field_name); + + // The two other main overrides for the subcomponent + void initialize_impl (const RunType run_type); + void finalize_impl (); + + // Computes total number of bytes needed for local variables + size_t requested_buffer_size_in_bytes() const; + + // Set local variables using memory provided by + // the ATMBufferManager + void init_buffers(const ATMBufferManager &buffer_manager); + + // Creates an helper field, not to be shared with the AD's FieldManager + Field create_helper_field (const std::string& name, + const FieldLayout& layout, + const std::string& grid_name, + const int ps=0); + + // Query if a local field exists + bool has_helper_field (const std::string& name) const { return m_helper_fields.find(name)!=m_helper_fields.end(); } + // Retrieve a helper field + Field get_helper_field (const std::string& name) const { return m_helper_fields.at(name); } + // Internal function to apply nudging at specific timescale + void apply_tendency(Field& base, const Field& next, const Real dt); + + std::shared_ptr m_grid; + // Keep track of field dimensions and the iteration count + int m_num_cols; + int m_num_levs; + int m_num_src_levs; + int m_timescale; + bool m_use_weights; + std::vector m_datafiles; + std::string m_static_vertical_pressure_file; + // add nudging weights for regional nudging update + std::string m_weights_file; + + SourcePresType m_src_pres_type; + + + // Some helper fields. + std::map m_helper_fields; + + std::vector m_fields_nudge; + + /* Nudge from coarse data */ + // if true, remap coarse data to fine grid + bool m_refine_remap; + // file containing coarse data mapping + std::string m_refine_remap_file; + // (refining) remapper object + std::shared_ptr m_refine_remapper; + // (refining) remapper vertical cutoff + Real m_refine_remap_vert_cutoff; + + util::TimeInterpolation m_time_interp; + + Buffer m_buffer; +}; // class Nudging + +} // namespace scream + +#endif // SCREAM_NUDGING_HPP diff --git a/components/eamxx/src/physics/nudging/nudging_functions.hpp b/components/eamxx/src/physics/nudging/nudging_functions.hpp deleted file mode 100644 index f304506b6004..000000000000 --- a/components/eamxx/src/physics/nudging/nudging_functions.hpp +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef NUDGING_FUNCTIONS_HPP -#define NUDGING_FUNCTIONS_HPP - -namespace scream{ -namespace nudging{ - -struct NudgingFunctions -{ - using mPack = ekat::Pack; - using KT = KokkosTypes; - - template - using view_2d = typename KT::template view_2d; - - template - using view_3d = typename KT::template view_3d; - - struct NudgingData - { - NudgingData() = default; - NudgingData& operator=(const NudgingData&) = default; - NudgingData(const int ncol_, const int nlev_) - { - init(ncol_,nlev_,true); - } - void init (const int ncol_, const int nlev_, bool allocate) - { - ncols=ncol_; - nlevs=nlev_; - if (allocate){ - T_mid = view_2d("",ncols,nlevs); - p_mid = view_2d("",ncols,nlevs); - qv = view_2d("",ncols,nlevs); - u = view_2d("",ncols,nlevs); - v = view_2d("",ncols,nlevs); - } - } - - int ncols; - int nlevs; - int time; - view_2d T_mid; - view_2d p_mid; - view_2d qv; - view_2d u; - view_2d v; - }; - -}; -}//end of nudging namespace -}//end of scream namespace - -#endif // NUDGING_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/nudging/tests/CMakeLists.txt b/components/eamxx/src/physics/nudging/tests/CMakeLists.txt index de516050ea96..0187b85d2476 100644 --- a/components/eamxx/src/physics/nudging/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/nudging/tests/CMakeLists.txt @@ -1,8 +1,8 @@ if (NOT SCREAM_BASELINES_ONLY) include(ScreamUtils) - set( NEED_LIBS scream_share nudging physics_share scream_io) - - CreateUnitTest(nudging_tests "nudging_tests.cpp" "${NEED_LIBS}" LABELS "physics_nudging" ) + CreateUnitTest(nudging_tests "nudging_tests.cpp" + LIBS nudging scream_io + LABELS "physics_nudging" ) endif() diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index 6f3a34b030f8..9851f2c72714 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -1,5 +1,5 @@ #include "catch2/catch.hpp" -#include "physics/nudging/atmosphere_nudging.hpp" +#include "physics/nudging/eamxx_nudging_process_interface.hpp" #include "share/grid/mesh_free_grids_manager.hpp" #include "share/io/scream_output_manager.hpp" @@ -26,9 +26,14 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { const int num_global_cols = ncols*comm.size(); + using vos_t = std::vector; ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_vertical_levels", nlevs); + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); auto gm = create_mesh_free_grids_manager(comm,gm_params); gm->build_grids(); @@ -48,7 +53,7 @@ TEST_CASE("nudging") { //It then runs then nudging module with the netcdf file as input to nudge //And then checks that the output fields of the nudging module match //what should be in the netcdf file - + using namespace ekat::units; using namespace ShortFieldTagsNames; using FL = FieldLayout; @@ -57,16 +62,17 @@ TEST_CASE("nudging") { util::TimeStamp t0 ({2000,1,1},{0,0,0}); ekat::Comm io_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. + const int packsize = SCREAM_PACK_SIZE; Int num_levs = 34; // Initialize the pio_subsystem for this test: - // MPI communicator group used for I/O. + // MPI communicator group used for I/O. // In our simple test we use MPI_COMM_WORLD, however a subset could be used. - MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); + MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); // Gather the initial PIO subsystem data creater by component coupler - scorpio::eam_init_pio_subsystem(fcomm); - - // First set up a field manager and grids manager to interact + scorpio::eam_init_pio_subsystem(fcomm); + + // First set up a field manager and grids manager to interact // with the output functions auto gm2 = create_gm(io_comm,3,num_levs); auto grid2 = gm2->get_grid("Point Grid"); @@ -76,7 +82,7 @@ TEST_CASE("nudging") { io_control.timestamp_of_last_write = t0; io_control.nsamples_since_last_write = 0; io_control.frequency_units = "nsteps"; - std::vector output_stamps; + std::vector output_stamps; const Int dt = 250; const Int max_steps = 12; @@ -96,29 +102,29 @@ TEST_CASE("nudging") { const int num_levs = grid2->get_num_vertical_levels(); // Create some fields for this fm - std::vector tag_h = {COL}; - std::vector tag_v = {LEV}; - std::vector tag_2d = {COL,LEV}; + std::vector tag_h = {COL}; + std::vector tag_v = {LEV}; + std::vector tag_2d = {COL,LEV}; + std::vector tag_2d_vec = {COL,CMP,LEV}; - std::vector dims_h = {num_lcols}; - std::vector dims_v = {num_levs}; - std::vector dims_2d = {num_lcols,num_levs}; + std::vector dims_h = {num_lcols}; + std::vector dims_v = {num_levs}; + std::vector dims_2d = {num_lcols,num_levs}; + std::vector dims_2d_vec = {num_lcols,2,num_levs}; const std::string& gn = grid2->name(); FieldIdentifier fid1("p_mid",FL{tag_2d,dims_2d},Pa,gn); FieldIdentifier fid2("T_mid",FL{tag_2d,dims_2d},K,gn); FieldIdentifier fid3("qv",FL{tag_2d,dims_2d},kg/kg,gn); - FieldIdentifier fid4("u",FL{tag_2d,dims_2d},m/s,gn); - FieldIdentifier fid5("v",FL{tag_2d,dims_2d},m/s,gn); + FieldIdentifier fidhw("horiz_winds",FL{tag_2d_vec,dims_2d_vec},m/s,gn); // Register fields with fm fm->registration_begins(); - fm->register_field(FR{fid1,"output"}); - fm->register_field(FR{fid2,"output"}); - fm->register_field(FR{fid3,"output"}); - fm->register_field(FR{fid4,"output"}); - fm->register_field(FR{fid5,"output"}); + fm->register_field(FR{fid1,"output",packsize}); + fm->register_field(FR{fid2,"output",packsize}); + fm->register_field(FR{fid3,"output",packsize}); + fm->register_field(FR{fidhw,"output",packsize}); fm->registration_ends(); // Initialize these fields @@ -131,34 +137,43 @@ TEST_CASE("nudging") { auto f3 = fm->get_field(fid3); auto f3_host = f3.get_view(); - auto f4 = fm->get_field(fid4); - auto f4_host = f4.get_view(); + auto fhw = fm->get_field(fidhw); + auto fhw_host = fhw.get_view(); - auto f5 = fm->get_field(fid5); - auto f5_host = f5.get_view(); - for (int ii=0;iiinit_fields_time_stamp(time); f1.sync_to_dev(); f2.sync_to_dev(); f3.sync_to_dev(); - f4.sync_to_dev(); - f5.sync_to_dev(); + fhw.sync_to_dev(); + + // Add subfields U and V to field manager + { + auto hw = fm->get_field("horiz_winds"); + const auto& fid = hw.get_header().get_identifier(); + const auto& layout = fid.get_layout(); + const int vec_dim = layout.get_vector_dim(); + const auto& units = fid.get_units(); + auto fU = hw.subfield("U",units,vec_dim,0); + auto fV = hw.subfield("V",units,vec_dim,1); + fm->add_field(fU); + fm->add_field(fV); + } // Set up parameter list control for output ekat::ParameterList params; params.set("filename_prefix","io_output_test"); params.set("Averaging Type","Instant"); params.set("Max Snapshots Per File",15); - std::vector fnames = {"T_mid","p_mid","qv","u","v"}; + std::vector fnames = {"T_mid","p_mid","qv","U","V"}; params.set>("Field Names",fnames); auto& params_sub = params.sublist("output_control"); params_sub.set("frequency_units","nsteps"); @@ -197,6 +212,17 @@ TEST_CASE("nudging") { } } break; + case 3: + { + auto v = f.get_view(); + for (int i=0; iget_grid("Physics"); ekat::ParameterList params_mid; std::string nudging_f = "io_output_test.INSTANT.nsteps_x1."\ "np1.2000-01-01-00000.nc"; - params_mid.set("Nudging_Filename",nudging_f); - auto nudging_mid = std::make_shared(io_comm,params_mid); + params_mid.set>("nudging_filename",{nudging_f}); + params_mid.set>("nudging_fields",{"T_mid","qv","U","V"}); + params_mid.set("use_nudging_weights",false); + std::shared_ptr nudging_mid = std::make_shared(io_comm,params_mid); nudging_mid->set_grids(gm); @@ -235,7 +263,7 @@ TEST_CASE("nudging") { for (const auto& req : nudging_mid->get_required_field_requests()) { Field f(req.fid); auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(1); + f_ap.request_allocation(packsize); f.allocate_view(); const auto name = f.name(); f.get_header().get_tracking().update_time_stamp(t0); @@ -252,45 +280,44 @@ TEST_CASE("nudging") { Field p_mid = input_fields["p_mid"]; Field T_mid = input_fields["T_mid"]; Field qv = input_fields["qv"]; - Field u = input_fields["u"]; - Field v = input_fields["v"]; - Field T_mid_o = output_fields["T_mid"]; - Field qv_mid_o = output_fields["qv"]; - Field u_o = output_fields["u"]; - Field v_o = output_fields["v"]; + Field hw = input_fields["horiz_winds"]; + Field T_mid_o = output_fields["T_mid"]; + Field qv_mid_o = output_fields["qv"]; + Field hw_o = output_fields["horiz_winds"]; + // Initialize memory buffer for all atm processes + auto memory_buffer = std::make_shared(); + memory_buffer->request_bytes(nudging_mid->requested_buffer_size_in_bytes()); + memory_buffer->allocate(); + nudging_mid->init_buffers(*memory_buffer); //fill data - //Don't fill T,qv,u,v because they will be nudged anyways + //Don't fill T,qv,u,v because they will be nudged anyways auto p_mid_v_h = p_mid.get_view(); for (int icol=0; icol(); - auto qv_h_o = qv_mid_o.get_view(); - auto u_h_o = u_o.get_view(); - auto v_h_o = v_o.get_view(); + auto qv_h_o = qv_mid_o.get_view(); + auto hw_h_o = hw_o.get_view(); nudging_mid->run(100); T_mid_o.sync_to_host(); qv_mid_o.sync_to_host(); - u_o.sync_to_host(); - v_o.sync_to_host(); + hw_o.sync_to_host(); for (int icol=0; icolfinalize(); - + } diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 052e0aa45346..fb90ca03be2c 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -3,8 +3,8 @@ set(P3_SRCS p3_ic_cases.cpp p3_iso_c.f90 ${SCREAM_BASE_DIR}/../eam/src/physics/p3/scream/micro_p3.F90 - atmosphere_microphysics.cpp - atmosphere_microphysics_run.cpp + eamxx_p3_process_interface.cpp + eamxx_p3_run.cpp ) if (NOT SCREAM_LIB_ONLY) @@ -61,18 +61,45 @@ if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos ) # P3 ETI SRCS endif() -add_library(p3 ${P3_SRCS}) -set_target_properties(p3 PROPERTIES - Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules -) -target_include_directories(p3 PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../share - ${CMAKE_CURRENT_BINARY_DIR}/modules - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/impl - ${SCREAM_BASE_DIR}/../eam/src/physics/cam -) -target_link_libraries(p3 physics_share scream_share) +# List of dispatch source files if monolithic kernels are off +set(P3_SK_SRCS + disp/p3_check_values_impl_disp.cpp + disp/p3_ice_sed_impl_disp.cpp + disp/p3_main_impl_part1_disp.cpp + disp/p3_main_impl_part3_disp.cpp + disp/p3_cloud_sed_impl_disp.cpp + disp/p3_main_impl_disp.cpp + disp/p3_main_impl_part2_disp.cpp + disp/p3_rain_sed_impl_disp.cpp + ) + +set(P3_LIBS "p3") +if (SCREAM_SMALL_KERNELS) + add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) +else() + add_library(p3 ${P3_SRCS}) + if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_BASELINES_ONLY) + add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) + # Always build shoc_sk with SCREAM_SMALL_KERNELS on + target_compile_definitions(p3_sk PUBLIC "SCREAM_SMALL_KERNELS") + list(APPEND P3_LIBS "p3_sk") + endif() +endif() + +target_compile_definitions(p3 PUBLIC EAMXX_HAS_P3) + +foreach (P3_LIB IN LISTS P3_LIBS) + set_target_properties(${P3_LIB} PROPERTIES + Fortran_MODULE_DIRECTORY ${P3_LIB}/modules + ) + target_include_directories(${P3_LIB} PUBLIC + ${CMAKE_CURRENT_BINARY_DIR}/${P3_LIB}/modules + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/impl + ${SCREAM_BASE_DIR}/../eam/src/physics/cam + ) + target_link_libraries(${P3_LIB} physics_share scream_share) +endforeach() # Ensure tables are present in the data dir if (SCREAM_DOUBLE_PRECISION) diff --git a/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp new file mode 100644 index 000000000000..cc13e99bc91d --- /dev/null +++ b/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp @@ -0,0 +1,35 @@ + + +#include "p3_functions.hpp" // for ETI only but harmless for GPU +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace p3 { + +template <> +void Functions +::check_values_disp(const uview_2d& qv, const uview_2d& temp, const Int& ktop, const Int& kbot, + const Int& timestepcount, const bool& force_abort, const Int& source_ind, + const uview_2d& col_loc, const Int& nj, const Int& nk) +{ + + using ExeSpace = typename KT::ExeSpace; + const Int nk_pack = ekat::npack(nk); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + + Kokkos::parallel_for( + "p3_check_values", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + + check_values(ekat::subview(qv, i), ekat::subview(temp, i), ktop, kbot, timestepcount, force_abort, 900, + team, ekat::subview(col_loc, i)); + + }); + +} + +} // namespace p3 +} // namespace scream + diff --git a/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp new file mode 100644 index 000000000000..8b755be4857f --- /dev/null +++ b/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp @@ -0,0 +1,63 @@ + +#include "p3_functions.hpp" // for ETI only but harmless for GPU +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace p3 { + +/* + * Implementation of p3 cloud sedimentation function. Clients should NOT #include + * this file, #include p3_functions.hpp instead. + */ + +template <> +void Functions +::cloud_sedimentation_disp( + const uview_2d& qc_incld, + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& cld_frac_l, + const uview_2d& acn, + const uview_2d& inv_dz, + const view_dnu_table& dnu, + const WorkspaceManager& workspace_mgr, + const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const Scalar& dt, const Scalar& inv_dt, const bool& do_predict_nc, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& nc_incld, + const uview_2d& mu_c, + const uview_2d& lamc, + const uview_2d& qc_tend, + const uview_2d& nc_tend, + const uview_1d& precip_liq_surf, + const uview_1d& nucleationPossible, + const uview_1d& hydrometeorsPresent) +{ + using ExeSpace = typename KT::ExeSpace; + const Int nk_pack = ekat::npack(nk); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + // p3_cloud_sedimentation loop + Kokkos::parallel_for( + "p3_cloud_sedimentation", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + auto workspace = workspace_mgr.get_workspace(team); + if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { + return; + } + + cloud_sedimentation( + ekat::subview(qc_incld, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(cld_frac_l, i), + ekat::subview(acn, i), ekat::subview(inv_dz, i), dnu, team, workspace, + nk, ktop, kbot, kdir, dt, inv_dt, do_predict_nc, + ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(nc_incld, i), ekat::subview(mu_c, i), ekat::subview(lamc, i), ekat::subview(qc_tend, i), + ekat::subview(nc_tend, i), + precip_liq_surf(i)); + }); + +} + +} // namespace p3 +} // namespace scream + diff --git a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp new file mode 100644 index 000000000000..e26d6fe18743 --- /dev/null +++ b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp @@ -0,0 +1,100 @@ + +#include "p3_functions.hpp" // for ETI only but harmless for GPU +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace p3 { + +template <> +void Functions +::ice_sedimentation_disp( + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& rhofaci, + const uview_2d& cld_frac_i, + const uview_2d& inv_dz, + const WorkspaceManager& workspace_mgr, + const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const Scalar& dt, const Scalar& inv_dt, + const uview_2d& qi, + const uview_2d& qi_incld, + const uview_2d& ni, + const uview_2d& ni_incld, + const uview_2d& qm, + const uview_2d& qm_incld, + const uview_2d& bm, + const uview_2d& bm_incld, + const uview_2d& qi_tend, + const uview_2d& ni_tend, + const view_ice_table& ice_table_vals, + const uview_1d& precip_ice_surf, + const uview_1d& nucleationPossible, + const uview_1d& hydrometeorsPresent) +{ + using ExeSpace = typename KT::ExeSpace; + const Int nk_pack = ekat::npack(nk); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + // p3_ice_sedimentation loop + Kokkos::parallel_for("p3_ice_sedimentation", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { + return; + } + auto workspace = workspace_mgr.get_workspace(team); + + // Ice sedimentation: (adaptive substepping) + ice_sedimentation( + ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(cld_frac_i, i), + ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, + ekat::subview(qi, i), ekat::subview(qi_incld, i), ekat::subview(ni, i), ekat::subview(ni_incld, i), + ekat::subview(qm, i), ekat::subview(qm_incld, i), ekat::subview(bm, i), ekat::subview(bm_incld, i), ekat::subview(qi_tend, i), ekat::subview(ni_tend, i), + ice_table_vals, precip_ice_surf(i)); + + }); +} + +template <> +void Functions +::homogeneous_freezing_disp( + const uview_2d& T_atm, + const uview_2d& inv_exner, + const uview_2d& latent_heat_fusion, + const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& qi, + const uview_2d& ni, + const uview_2d& qm, + const uview_2d& bm, + const uview_2d& th_atm, + const uview_1d& nucleationPossible, + const uview_1d& hydrometeorsPresent) +{ + using ExeSpace = typename KT::ExeSpace; + const Int nk_pack = ekat::npack(nk); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + // p3_cloud_sedimentation loop + Kokkos::parallel_for( + "p3_homogeneous", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { + return; + } + + // homogeneous freezing of cloud and rain + homogeneous_freezing( + ekat::subview(T_atm, i), ekat::subview(inv_exner, i), ekat::subview(latent_heat_fusion, i), team, nk, ktop, kbot, kdir, + ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), + ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(th_atm, i)); + + }); +} + +} // namespace p3 +} // namespace scream + diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp new file mode 100644 index 000000000000..5b075439aa91 --- /dev/null +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -0,0 +1,325 @@ + +#include "p3_functions.hpp" // for ETI only but harmless for GPU +#include "physics/share/physics_functions.hpp" // also for ETI not on GPUs +#include "physics/share/physics_saturation_impl.hpp" +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace p3 { + +/* + * Implementation of p3 main function. Clients should NOT #include + * this file, #include p3_functions.hpp instead. + */ + +template <> +void Functions +::p3_main_init_disp( + const Int& nj, const Int& nk_pack, const uview_2d& cld_frac_i, const uview_2d& cld_frac_l, + const uview_2d& cld_frac_r, const uview_2d& inv_exner, const uview_2d& th_atm, + const uview_2d& dz, const uview_2d& diag_equiv_reflectivity, const uview_2d& ze_ice, + const uview_2d& ze_rain, const uview_2d& diag_eff_radius_qc, const uview_2d& diag_eff_radius_qi, + const uview_2d& diag_eff_radius_qr, + const uview_2d& inv_cld_frac_i, const uview_2d& inv_cld_frac_l, const uview_2d& inv_cld_frac_r, + const uview_2d& exner, const uview_2d& T_atm, const uview_2d& qv, const uview_2d& inv_dz, + const uview_1d& precip_liq_surf, const uview_1d& precip_ice_surf, + const uview_2d& mu_r, const uview_2d& lamr, const uview_2d& logn0r, const uview_2d& nu, + const uview_2d& cdist, const uview_2d& cdist1, const uview_2d& cdistr, + const uview_2d& qc_incld, const uview_2d& qr_incld, const uview_2d& qi_incld, + const uview_2d& qm_incld, const uview_2d& nc_incld, const uview_2d& nr_incld, const uview_2d& ni_incld, + const uview_2d& bm_incld, const uview_2d& inv_rho, const uview_2d& prec, const uview_2d& rho, const uview_2d& rhofacr, + const uview_2d& rhofaci, const uview_2d& acn, const uview_2d& qv_sat_l, const uview_2d& qv_sat_i, const uview_2d& sup, + const uview_2d& qv_supersat_i, const uview_2d& qtend_ignore, const uview_2d& ntend_ignore, const uview_2d& mu_c, + const uview_2d& lamc, const uview_2d& rho_qi, const uview_2d& qv2qi_depos_tend, const uview_2d& precip_total_tend, + const uview_2d& nevapr, const uview_2d& precip_liq_flux, const uview_2d& precip_ice_flux) +{ + using ExeSpace = typename KT::ExeSpace; + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + Kokkos::parallel_for("p3_main_init", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + precip_liq_surf(i) = 0; + precip_ice_surf(i) = 0; + + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nk_pack), [&] (Int k) { + diag_equiv_reflectivity(i,k) = -99; + ze_ice(i,k) = 1.e-22; + ze_rain(i,k) = 1.e-22; + diag_eff_radius_qc(i,k) = 10.e-6; + diag_eff_radius_qi(i,k) = 25.e-6; + diag_eff_radius_qr(i,k) = 500.e-6; + inv_cld_frac_i(i,k) = 1 / cld_frac_i(i,k); + inv_cld_frac_l(i,k) = 1 / cld_frac_l(i,k); + inv_cld_frac_r(i,k) = 1 / cld_frac_r(i,k); + exner(i,k) = 1 / inv_exner(i,k); + T_atm(i,k) = th_atm(i,k) * exner(i,k); + qv(i,k) = max(qv(i,k), 0); + inv_dz(i,k) = 1 / dz(i,k); + mu_r(i,k) = 0.; + lamr(i,k) = 0.; + logn0r(i,k) = 0.; + nu(i,k) = 0.; + cdist(i,k) = 0.; + cdist1(i,k) = 0.; + cdistr(i,k) = 0.; + qc_incld(i,k) = 0.; + qr_incld(i,k) = 0.; + qi_incld(i,k) = 0.; + qm_incld(i,k) = 0.; + nc_incld(i,k) = 0.; + nr_incld(i,k) = 0.; + ni_incld(i,k) = 0.; + bm_incld(i,k) = 0.; + inv_rho(i,k) = 0.; + prec(i,k) = 0.; + rho(i,k) = 0.; + rhofacr(i,k) = 0.; + rhofaci(i,k) = 0.; + acn(i,k) = 0.; + qv_sat_l(i,k) = 0.; + qv_sat_i(i,k) = 0.; + sup(i,k) = 0.; + qv_supersat_i(i,k) = 0.; + qtend_ignore(i,k) = 0.; + ntend_ignore(i,k) = 0.; + mu_c(i,k) = 0.; + lamc(i,k) = 0.; + rho_qi(i,k) = 0.; + qv2qi_depos_tend(i,k) = 0.; + precip_total_tend(i,k) = 0.; + nevapr(i,k) = 0.; + precip_liq_flux(i,k) = 0.; + precip_ice_flux(i,k) = 0.; + }); + }); +} + +template <> +Int Functions +::p3_main_internal_disp( + const P3Runtime& runtime_options, + const P3PrognosticState& prognostic_state, + const P3DiagnosticInputs& diagnostic_inputs, + const P3DiagnosticOutputs& diagnostic_outputs, + const P3Infrastructure& infrastructure, + const P3HistoryOnly& history_only, + const P3LookupTables& lookup_tables, + const WorkspaceManager& workspace_mgr, + Int nj, + Int nk) +{ + using ExeSpace = typename KT::ExeSpace; + + view_2d latent_heat_sublim("latent_heat_sublim", nj, nk), latent_heat_vapor("latent_heat_vapor", nj, nk), latent_heat_fusion("latent_heat_fusion", nj, nk); + + get_latent_heat(nj, nk, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion); + + const Int nk_pack = ekat::npack(nk); + + // load constants into local vars + const Scalar inv_dt = 1 / infrastructure.dt; + constexpr Int kdir = -1; + const Int ktop = kdir == -1 ? 0 : nk-1; + const Int kbot = kdir == -1 ? nk-1 : 0; + constexpr bool debug_ABORT = false; + + // per-column bools + view_1d nucleationPossible("nucleationPossible", nj); + view_1d hydrometeorsPresent("hydrometeorsPresent", nj); + + // + // Create temporary variables needed for p3 + // + view_2d + mu_r("mu_r", nj, nk_pack), // shape parameter of rain + T_atm("T_atm", nj, nk_pack), // temperature at the beginning of the microphysics step [K] + + // 2D size distribution and fallspeed parameters + lamr("lamr", nj, nk_pack), logn0r("logn0r", nj, nk_pack), nu("nu", nj, nk_pack), + cdist("cdist", nj, nk_pack), cdist1("cdist1", nj, nk_pack), cdistr("cdistr", nj, nk_pack), + + // Variables needed for in-cloud calculations + // Inverse cloud fractions (1/cld) + inv_cld_frac_i("inv_cld_frac_i", nj, nk_pack), inv_cld_frac_l("inv_cld_frac_l", nj, nk_pack), inv_cld_frac_r("inv_cld_frac_r", nj, nk_pack), + // In cloud mass-mixing ratios + qc_incld("qc_incld", nj, nk_pack), qr_incld("qr_incld", nj, nk_pack), qi_incld("qi_incld", nj, nk_pack), qm_incld("qm_incld", nj, nk_pack), + // In cloud number concentrations + nc_incld("nc_incld", nj, nk_pack), nr_incld("nr_incld", nj, nk_pack), ni_incld("ni_incld", nj, nk_pack), bm_incld("bm_incld", nj, nk_pack), + + // Other + inv_dz("inv_dz", nj, nk_pack), inv_rho("inv_rho", nj, nk_pack), ze_ice("ze_ice", nj, nk_pack), ze_rain("ze_rain", nj, nk_pack), + prec("prec", nj, nk_pack), rho("rho", nj, nk_pack), rhofacr("rhofacr", nj, nk_pack), rhofaci("rhofaci", nj, nk_pack), + acn("acn", nj, nk_pack), qv_sat_l("qv_sat", nj, nk_pack), qv_sat_i("qv_sat_i", nj, nk_pack), sup("sup", nj, nk_pack), + qv_supersat_i("qv_supersat", nj, nk_pack), tmparr2("tmparr2", nj, nk_pack), exner("exner", nj, nk_pack), + diag_equiv_reflectivity("diag_equiv_ref", nj, nk_pack), diag_vm_qi("diag_vm_qi", nj, nk_pack), diag_diam_qi("diag_diam_qi", nj, nk_pack), + pratot("pratot", nj, nk_pack), prctot("prctot", nj, nk_pack), + + // p3_tend_out, may not need these + qtend_ignore("qtend_ignore", nj, nk_pack), ntend_ignore("ntend_ignore", nj, nk_pack), + + // Variables still used in F90 but removed from C++ interface + mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), precip_total_tend("precip_total_tend", nj, nk_pack), + nevapr("nevapr", nj, nk_pack), qr_evap_tend("qr_evap_tend", nj, nk_pack), + + // cloud sedimentation + v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), + + // ice sedimentation + v_qit("v_qit", nj, nk_pack), v_nit("v_nit", nj, nk_pack), flux_nit("flux_nit", nj, nk_pack), flux_bir("flux_bir", nj, nk_pack), + flux_qir("flux_qir", nj, nk_pack), flux_qit("flux_qit", nj, nk_pack), + + // rain sedimentation + v_qr("v_qr", nj, nk_pack), v_nr("v_nr", nj, nk_pack); + + // Get views of all inputs + auto pres = diagnostic_inputs.pres; + auto dz = diagnostic_inputs.dz; + auto nc_nuceat_tend = diagnostic_inputs.nc_nuceat_tend; + auto nccn_prescribed = diagnostic_inputs.nccn; + auto ni_activated = diagnostic_inputs.ni_activated; + auto inv_qc_relvar = diagnostic_inputs.inv_qc_relvar; + auto dpres = diagnostic_inputs.dpres; + auto inv_exner = diagnostic_inputs.inv_exner; + auto cld_frac_i = diagnostic_inputs.cld_frac_i; + auto cld_frac_l = diagnostic_inputs.cld_frac_l; + auto cld_frac_r = diagnostic_inputs.cld_frac_r; + auto col_location = infrastructure.col_location; + auto qc = prognostic_state.qc; + auto nc = prognostic_state.nc; + auto qr = prognostic_state.qr; + auto nr = prognostic_state.nr; + auto qi = prognostic_state.qi; + auto qm = prognostic_state.qm; + auto ni = prognostic_state.ni; + auto bm = prognostic_state.bm; + auto qv = prognostic_state.qv; + auto th = prognostic_state.th; + auto diag_eff_radius_qc = diagnostic_outputs.diag_eff_radius_qc; + auto diag_eff_radius_qi = diagnostic_outputs.diag_eff_radius_qi; + auto diag_eff_radius_qr = diagnostic_outputs.diag_eff_radius_qr; + auto qv2qi_depos_tend = diagnostic_outputs.qv2qi_depos_tend; + auto rho_qi = diagnostic_outputs.rho_qi; + auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; + auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; + auto qv_prev = diagnostic_inputs.qv_prev; + auto t_prev = diagnostic_inputs.t_prev; + auto liq_ice_exchange = history_only.liq_ice_exchange; + auto vap_liq_exchange = history_only.vap_liq_exchange; + auto vap_ice_exchange = history_only.vap_ice_exchange; + + // we do not want to measure init stuff + auto start = std::chrono::steady_clock::now(); + + // initialize + p3_main_init_disp( + nj, nk_pack, cld_frac_i, cld_frac_l, cld_frac_r, inv_exner, th, dz, diag_equiv_reflectivity, + ze_ice, ze_rain, diag_eff_radius_qc, diag_eff_radius_qi, diag_eff_radius_qr, + inv_cld_frac_i, inv_cld_frac_l, inv_cld_frac_r, exner, T_atm, qv, inv_dz, + diagnostic_outputs.precip_liq_surf, diagnostic_outputs.precip_ice_surf, + mu_r, lamr, logn0r, nu, cdist, cdist1, cdistr, + qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, + inv_rho, prec, rho, rhofacr, rhofaci, acn, qv_sat_l, qv_sat_i, sup, qv_supersat_i, + qtend_ignore, ntend_ignore, mu_c, lamc, rho_qi, qv2qi_depos_tend, precip_total_tend, + nevapr, precip_liq_flux, precip_ice_flux); + + p3_main_part1_disp( + nj, nk, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, + pres, dpres, dz, nc_nuceat_tend, nccn_prescribed, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, + inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, + T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, + rhofaci, acn, qv, th, qc, nc, qr, nr, qi, ni, qm, + bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, + ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent); + + // ------------------------------------------------------------------------------------------ + // main k-loop (for processes): + + p3_main_part2_disp( + nj, nk, runtime_options.max_total_ni, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, inv_dt, + lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, + lookup_tables.revap_table_vals, pres, dpres, dz, nc_nuceat_tend, inv_exner, + exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, + cld_frac_l, cld_frac_r, qv_prev, t_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, + qv, th, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, + latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, + nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, + mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, + vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, + pratot, prctot, nucleationPossible, hydrometeorsPresent); + + //NOTE: At this point, it is possible to have negative (but small) nc, nr, ni. This is not + // a problem; those values get clipped to zero in the sedimentation section (if necessary). + // (This is not done above simply for efficiency purposes.) + + // ----------------------------------------------------------------------------------------- + // End of main microphysical processes section + // ========================================================================================= + + // ==========================================================================================! + // Sedimentation: + + // Cloud sedimentation: (adaptive substepping) + cloud_sedimentation_disp( + qc_incld, rho, inv_rho, cld_frac_l, acn, inv_dz, lookup_tables.dnu_table_vals, workspace_mgr, + nj, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, infrastructure.predictNc, + qc, nc, nc_incld, mu_c, lamc, qtend_ignore, ntend_ignore, + diagnostic_outputs.precip_liq_surf, nucleationPossible, hydrometeorsPresent); + + + // Rain sedimentation: (adaptive substepping) + rain_sedimentation_disp( + rho, inv_rho, rhofacr, cld_frac_r, inv_dz, qr_incld, workspace_mgr, + lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, nj, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, qr, + nr, nr_incld, mu_r, lamr, precip_liq_flux, qtend_ignore, ntend_ignore, + diagnostic_outputs.precip_liq_surf, nucleationPossible, hydrometeorsPresent); + + // Ice sedimentation: (adaptive substepping) + ice_sedimentation_disp( + rho, inv_rho, rhofaci, cld_frac_i, inv_dz, workspace_mgr, nj, nk, ktop, kbot, + kdir, infrastructure.dt, inv_dt, qi, qi_incld, ni, ni_incld, + qm, qm_incld, bm, bm_incld, qtend_ignore, ntend_ignore, + lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf, nucleationPossible, hydrometeorsPresent); + + // homogeneous freezing f cloud and rain + homogeneous_freezing_disp( + T_atm, inv_exner, latent_heat_fusion, nj, nk, ktop, kbot, kdir, qc, nc, qr, nr, qi, + ni, qm, bm, th, nucleationPossible, hydrometeorsPresent); + + // + // final checks to ensure consistency of mass/number + // and compute diagnostic fields for output + // + p3_main_part3_disp( + nj, nk_pack, runtime_options.max_total_ni, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, + rho, inv_rho, rhofaci, qv, th, qc, nc, qr, nr, qi, ni, + qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, + vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, + rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent); + + // + // merge ice categories with similar properties + + // note: this should be relocated to above, such that the diagnostic + // ice properties are computed after merging + + // PMC nCat deleted nCat>1 stuff + +#ifndef NDEBUG + Kokkos::parallel_for( + Kokkos::MDRangePolicy>({0, 0}, {nj, nk_pack}), KOKKOS_LAMBDA (int i, int k) { + tmparr2(i,k) = th(i,k) * exner(i,k); + }); + check_values_disp(qv, tmparr2, ktop, kbot, infrastructure.it, debug_ABORT, 900, col_location, nj, nk); +#endif + Kokkos::fence(); + + auto finish = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(finish - start); + return duration.count(); +} + +} // namespace p3 +} // namespace scream + diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp new file mode 100644 index 000000000000..a9c0e5fe1445 --- /dev/null +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp @@ -0,0 +1,93 @@ + +#include "p3_functions.hpp" // for ETI only but harmless for GPU +#include "physics/share/physics_functions.hpp" // also for ETI not on GPUs +#include "physics/share/physics_saturation_impl.hpp" + +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace p3 { + +/* + * Implementation of p3 main function. Clients should NOT #include + * this file, #include p3_functions.hpp instead. + */ + +template <> +void Functions +::p3_main_part1_disp( + const Int& nj, + const Int& nk, + const bool& predictNc, + const bool& prescribedCCN, + const Scalar& dt, + const uview_2d& pres, + const uview_2d& dpres, + const uview_2d& dz, + const uview_2d& nc_nuceat_tend, + const uview_2d& nccn_prescribed, + const uview_2d& inv_exner, + const uview_2d& exner, + const uview_2d& inv_cld_frac_l, + const uview_2d& inv_cld_frac_i, + const uview_2d& inv_cld_frac_r, + const uview_2d& latent_heat_vapor, + const uview_2d& latent_heat_sublim, + const uview_2d& latent_heat_fusion, + const uview_2d& T_atm, + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& qv_sat_l, + const uview_2d& qv_sat_i, + const uview_2d& qv_supersat_i, + const uview_2d& rhofacr, + const uview_2d& rhofaci, + const uview_2d& acn, + const uview_2d& qv, + const uview_2d& th_atm, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& qi, + const uview_2d& ni, + const uview_2d& qm, + const uview_2d& bm, + const uview_2d& qc_incld, + const uview_2d& qr_incld, + const uview_2d& qi_incld, + const uview_2d& qm_incld, + const uview_2d& nc_incld, + const uview_2d& nr_incld, + const uview_2d& ni_incld, + const uview_2d& bm_incld, + const uview_1d& nucleationPossible, + const uview_1d& hydrometeorsPresent) +{ + using ExeSpace = typename KT::ExeSpace; + const Int nk_pack = ekat::npack(nk); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + // p3_cloud_sedimentation loop + Kokkos::parallel_for("p3_main_part1", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + + p3_main_part1( + team, nk, predictNc, prescribedCCN, dt, + ekat::subview(pres, i), ekat::subview(dpres, i), ekat::subview(dz, i), ekat::subview(nc_nuceat_tend, i), + ekat::subview(nccn_prescribed, i), ekat::subview(inv_exner, i), ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), + ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), + ekat::subview(latent_heat_fusion, i), ekat::subview(T_atm, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), + ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), ekat::subview(rhofaci, i), ekat::subview(acn, i), + ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), + ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(qc_incld, i), ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), + ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), + nucleationPossible(i), hydrometeorsPresent(i)); + + }); +} + +} // namespace p3 +} // namespace scream + diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp new file mode 100644 index 000000000000..02073bd4f5af --- /dev/null +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -0,0 +1,133 @@ + +#include "p3_functions.hpp" // for ETI only but harmless for GPU +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace p3 { + +/* + * Implementation of p3 main function. Clients should NOT #include + * this file, #include p3_functions.hpp instead. + */ + +template <> +void Functions +::p3_main_part2_disp( + const Int& nj, + const Int& nk, + const Scalar& max_total_ni, + const bool& predictNc, + const bool& do_prescribed_CCN, + const Scalar& dt, + const Scalar& inv_dt, + const view_dnu_table& dnu_table_vals, + const view_ice_table& ice_table_vals, + const view_collect_table& collect_table_vals, + const view_2d_table& revap_table_vals, + const uview_2d& pres, + const uview_2d& dpres, + const uview_2d& dz, + const uview_2d& nc_nuceat_tend, + const uview_2d& inv_exner, + const uview_2d& exner, + const uview_2d& inv_cld_frac_l, + const uview_2d& inv_cld_frac_i, + const uview_2d& inv_cld_frac_r, + const uview_2d& ni_activated, + const uview_2d& inv_qc_relvar, + const uview_2d& cld_frac_i, + const uview_2d& cld_frac_l, + const uview_2d& cld_frac_r, + const uview_2d& qv_prev, + const uview_2d& t_prev, + const uview_2d& T_atm, + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& qv_sat_l, + const uview_2d& qv_sat_i, + const uview_2d& qv_supersat_i, + const uview_2d& rhofacr, + const uview_2d& rhofaci, + const uview_2d& acn, + const uview_2d& qv, + const uview_2d& th_atm, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& qi, + const uview_2d& ni, + const uview_2d& qm, + const uview_2d& bm, + const uview_2d& latent_heat_vapor, + const uview_2d& latent_heat_sublim, + const uview_2d& latent_heat_fusion, + const uview_2d& qc_incld, + const uview_2d& qr_incld, + const uview_2d& qi_incld, + const uview_2d& qm_incld, + const uview_2d& nc_incld, + const uview_2d& nr_incld, + const uview_2d& ni_incld, + const uview_2d& bm_incld, + const uview_2d& mu_c, + const uview_2d& nu, + const uview_2d& lamc, + const uview_2d& cdist, + const uview_2d& cdist1, + const uview_2d& cdistr, + const uview_2d& mu_r, + const uview_2d& lamr, + const uview_2d& logn0r, + const uview_2d& qv2qi_depos_tend, + const uview_2d& precip_total_tend, + const uview_2d& nevapr, + const uview_2d& qr_evap_tend, + const uview_2d& vap_liq_exchange, + const uview_2d& vap_ice_exchange, + const uview_2d& liq_ice_exchange, + const uview_2d& pratot, + const uview_2d& prctot, + const uview_1d& nucleationPossible, + const uview_1d& hydrometeorsPresent) +{ + using ExeSpace = typename KT::ExeSpace; + const Int nk_pack = ekat::npack(nk); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + // p3_cloud_sedimentation loop + Kokkos::parallel_for( + "p3_main_part2_disp", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { + return; + } + + // ------------------------------------------------------------------------------------------ + // main k-loop (for processes): + p3_main_part2( + team, nk_pack, max_total_ni, predictNc, do_prescribed_CCN, dt, inv_dt, + dnu_table_vals, ice_table_vals, collect_table_vals, revap_table_vals, + ekat::subview(pres, i), ekat::subview(dpres, i), ekat::subview(dz, i), ekat::subview(nc_nuceat_tend, i), ekat::subview(inv_exner, i), + ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), + ekat::subview(ni_activated, i), ekat::subview(inv_qc_relvar, i), ekat::subview(cld_frac_i, i), ekat::subview(cld_frac_l, i), + ekat::subview(cld_frac_r, i), ekat::subview(qv_prev, i), ekat::subview(t_prev, i), ekat::subview(T_atm, i), ekat::subview(rho, i), + ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), + ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), + ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), + ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), ekat::subview(latent_heat_fusion, i), ekat::subview(qc_incld, i), + ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), + ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(cdist, i), + ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), + ekat::subview(qv2qi_depos_tend, i), ekat::subview(precip_total_tend, i), + ekat::subview(nevapr, i), ekat::subview(qr_evap_tend, i), ekat::subview(vap_liq_exchange, i), ekat::subview(vap_ice_exchange, i), ekat::subview(liq_ice_exchange, i), + ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk); + + if (!hydrometeorsPresent(i)) return; + }); +} + +} // namespace p3 +} // namespace scream + diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp new file mode 100644 index 000000000000..8e442cf2fc53 --- /dev/null +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp @@ -0,0 +1,92 @@ + +#include "p3_functions.hpp" // for ETI only but harmless for GPU +#include "physics/share/physics_functions.hpp" // also for ETI not on GPUs +#include "physics/share/physics_saturation_impl.hpp" + +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace p3 { + +/* + * Implementation of p3 main function. Clients should NOT #include + * this file, #include p3_functions.hpp instead. + */ + +template <> +void Functions +::p3_main_part3_disp( + const Int& nj, + const Int& nk_pack, + const Scalar& max_total_ni, + const view_dnu_table& dnu_table_vals, + const view_ice_table& ice_table_vals, + const uview_2d& inv_exner, + const uview_2d& cld_frac_l, + const uview_2d& cld_frac_r, + const uview_2d& cld_frac_i, + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& rhofaci, + const uview_2d& qv, + const uview_2d& th_atm, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& qi, + const uview_2d& ni, + const uview_2d& qm, + const uview_2d& bm, + const uview_2d& latent_heat_vapor, + const uview_2d& latent_heat_sublim, + const uview_2d& mu_c, + const uview_2d& nu, + const uview_2d& lamc, + const uview_2d& mu_r, + const uview_2d& lamr, + const uview_2d& vap_liq_exchange, + const uview_2d& ze_rain, + const uview_2d& ze_ice, + const uview_2d& diag_vm_qi, + const uview_2d& diag_eff_radius_qi, + const uview_2d& diag_diam_qi, + const uview_2d& rho_qi, + const uview_2d& diag_equiv_reflectivity, + const uview_2d& diag_eff_radius_qc, + const uview_2d& diag_eff_radius_qr, + const uview_1d& nucleationPossible, + const uview_1d& hydrometeorsPresent) +{ + using ExeSpace = typename KT::ExeSpace; + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + // p3_cloud_sedimentation loop + Kokkos::parallel_for( + "p3_main_part3_disp", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { + return; + } + + // + // final checks to ensure consistency of mass/number + // and compute diagnostic fields for output + // + p3_main_part3( + team, nk_pack, max_total_ni, dnu_table_vals, ice_table_vals, ekat::subview(inv_exner, i), ekat::subview(cld_frac_l, i), ekat::subview(cld_frac_r, i), + ekat::subview(cld_frac_i, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(qv, i), + ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), + ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), + ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), + ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), + ekat::subview(diag_diam_qi, i), ekat::subview(rho_qi, i), ekat::subview(diag_equiv_reflectivity, i), ekat::subview(diag_eff_radius_qc, i), + ekat::subview(diag_eff_radius_qr, i)); + + }); +} + +} // namespace p3 +} // namespace scream + diff --git a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp new file mode 100644 index 000000000000..6a9a36b4b353 --- /dev/null +++ b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp @@ -0,0 +1,57 @@ + +#include "p3_functions.hpp" // for ETI only but harmless for GPU +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace p3 { + +template <> +void Functions +::rain_sedimentation_disp( + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& rhofacr, + const uview_2d& cld_frac_r, + const uview_2d& inv_dz, + const uview_2d& qr_incld, + const WorkspaceManager& workspace_mgr, + const view_2d_table& vn_table_vals, const view_2d_table& vm_table_vals, + const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const Scalar& dt, const Scalar& inv_dt, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& nr_incld, + const uview_2d& mu_r, + const uview_2d& lamr, + const uview_2d& precip_liq_flux, + const uview_2d& qr_tend, + const uview_2d& nr_tend, + const uview_1d& precip_liq_surf, + const uview_1d& nucleationPossible, + const uview_1d& hydrometeorsPresent) +{ + using ExeSpace = typename KT::ExeSpace; + const Int nk_pack = ekat::npack(nk); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + // p3_rain_sedimentation loop + Kokkos::parallel_for("p3_rain_sed_disp", + policy, KOKKOS_LAMBDA(const MemberType& team) { + + const Int i = team.league_rank(); + auto workspace = workspace_mgr.get_workspace(team); + if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { + return; + } + + // Rain sedimentation: (adaptive substepping) + rain_sedimentation( + ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofacr, i), ekat::subview(cld_frac_r, i), + ekat::subview(inv_dz, i), ekat::subview(qr_incld, i), + team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, + ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), + ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), + ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i)); + }); + +} +} // namespace p3 +} // namespace scream diff --git a/components/eamxx/src/physics/p3/atmosphere_microphysics.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp similarity index 91% rename from components/eamxx/src/physics/p3/atmosphere_microphysics.cpp rename to components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index fcad5f1fc2ee..04b304e2691c 100644 --- a/components/eamxx/src/physics/p3/atmosphere_microphysics.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -1,4 +1,4 @@ -#include "physics/p3/atmosphere_microphysics.hpp" +#include "physics/p3/eamxx_p3_process_interface.hpp" #include "share/property_checks/field_within_interval_check.hpp" #include "share/property_checks/field_lower_bound_check.hpp" // Needed for p3_init, the only F90 code still used. @@ -45,8 +45,8 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m infrastructure.ite = m_num_cols-1; infrastructure.kts = 0; infrastructure.kte = m_num_levs-1; - infrastructure.predictNc = m_params.get("do_predict_nc",true); - infrastructure.prescribedCCN = m_params.get("do_prescribed_ccn",true); + infrastructure.predictNc = m_params.get("do_predict_nc",true); + infrastructure.prescribedCCN = m_params.get("do_prescribed_ccn",true); // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -63,8 +63,11 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m constexpr int ps = Pack::n; - // These variables are needed by the interface, but not actually passed to p3_main. + // These variables are needed by the interface, but not actually passed to p3_main. add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name, ps); + +//should we use one pressure only, wet/full? + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); add_field("p_dry_mid", scalar3d_layout_mid, Pa, grid_name, ps); add_field ("T_mid", scalar3d_layout_mid, K, grid_name, ps); // T_mid is the only one of these variables that is also updated. @@ -86,6 +89,7 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m } add_field("ni_activated", scalar3d_layout_mid, 1/kg, grid_name, ps); add_field("inv_qc_relvar", scalar3d_layout_mid, Q*Q, grid_name, ps); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name, ps); add_field ("qv_prev_micro_step", scalar3d_layout_mid, Q, grid_name, ps); add_field ("T_prev_micro_step", scalar3d_layout_mid, K, grid_name, ps); @@ -95,6 +99,7 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m add_field("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name, ps); add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name, ps); + add_field("eff_radius_qr", scalar3d_layout_mid, micron, grid_name, ps); // History Only: (all fields are just outputs and are really only meant for I/O purposes) // TODO: These should be averaged over subcycle as well. But there is no simple mechanism @@ -103,6 +108,7 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m add_field("micro_liq_ice_exchange", scalar3d_layout_mid, Q, grid_name, ps); add_field("micro_vap_liq_exchange", scalar3d_layout_mid, Q, grid_name, ps); add_field("micro_vap_ice_exchange", scalar3d_layout_mid, Q, grid_name, ps); + add_field("rainfrac", scalar3d_layout_mid, nondim, grid_name, ps); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { @@ -167,8 +173,6 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) s_mem += m_buffer.cld_frac_l.size(); m_buffer.cld_frac_i = decltype(m_buffer.cld_frac_i)(s_mem, m_num_cols, nk_pack); s_mem += m_buffer.cld_frac_i.size(); - m_buffer.cld_frac_r = decltype(m_buffer.cld_frac_r)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.cld_frac_r.size(); m_buffer.dz = decltype(m_buffer.dz)(s_mem, m_num_cols, nk_pack); s_mem += m_buffer.dz.size(); m_buffer.qv2qi_depos_tend = decltype(m_buffer.qv2qi_depos_tend)(s_mem, m_num_cols, nk_pack); @@ -198,6 +202,8 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) // ========================================================================================= void P3Microphysics::initialize_impl (const RunType /* run_type */) { + // Gather runtime options + runtime_options.max_total_ni = m_params.get("max_total_ni"); // Set property checks for fields in this process add_invariant_check(get_field_out("T_mid"),m_grid,100.0,500.0,false); add_invariant_check(get_field_out("qv"),m_grid,1e-13,0.2,true); @@ -216,6 +222,7 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) add_postcondition_check(get_field_out("precip_ice_surf_mass"),m_grid,0.0,false); add_postcondition_check(get_field_out("eff_radius_qc"),m_grid,0.0,1.0e2,false); add_postcondition_check(get_field_out("eff_radius_qi"),m_grid,0.0,5.0e3,false); + add_postcondition_check(get_field_out("eff_radius_qr"),m_grid,0.0,5.0e3,false); // Initialize p3 p3::p3_init(/* write_tables = */ false, @@ -226,8 +233,10 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) // variables a local view is constructed. const Int nk_pack = ekat::npack(m_num_levs); const Int nk_pack_p1 = ekat::npack(m_num_levs+1); - const auto& pmid = get_field_in("p_dry_mid").get_view(); - const auto& pseudo_density = get_field_in("pseudo_density_dry").get_view(); + const auto& pmid = get_field_in("p_mid").get_view(); + const auto& pmid_dry = get_field_in("p_dry_mid").get_view(); + const auto& pseudo_density = get_field_in("pseudo_density").get_view(); + const auto& pseudo_density_dry = get_field_in("pseudo_density_dry").get_view(); const auto& T_atm = get_field_out("T_mid").get_view(); const auto& cld_frac_t = get_field_in("cldfrac_tot").get_view(); const auto& qv = get_field_out("qv").get_view(); @@ -242,17 +251,18 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) auto qv_prev = get_field_out("qv_prev_micro_step").get_view(); const auto& precip_liq_surf_mass = get_field_out("precip_liq_surf_mass").get_view(); const auto& precip_ice_surf_mass = get_field_out("precip_ice_surf_mass").get_view(); + auto cld_frac_r = get_field_out("rainfrac").get_view(); // Alias local variables from temporary buffer auto inv_exner = m_buffer.inv_exner; auto th_atm = m_buffer.th_atm; auto cld_frac_l = m_buffer.cld_frac_l; auto cld_frac_i = m_buffer.cld_frac_i; - auto cld_frac_r = m_buffer.cld_frac_r; auto dz = m_buffer.dz; // -- Set values for the pre-amble structure - p3_preproc.set_variables(m_num_cols,nk_pack,pmid,pseudo_density,T_atm,cld_frac_t, + p3_preproc.set_variables(m_num_cols,nk_pack,pmid,pmid_dry,pseudo_density,pseudo_density_dry, + T_atm,cld_frac_t, qv, qc, nc, qr, nr, qi, qm, ni, bm, qv_prev, inv_exner, th_atm, cld_frac_l, cld_frac_i, cld_frac_r, dz); // --Prognostic State Variables: @@ -275,8 +285,10 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) } diag_inputs.ni_activated = get_field_in("ni_activated").get_view(); diag_inputs.inv_qc_relvar = get_field_in("inv_qc_relvar").get_view(); + + // P3 will use dry pressure for dry qv_sat diag_inputs.pres = get_field_in("p_dry_mid").get_view(); - diag_inputs.dpres = p3_preproc.pseudo_density; + diag_inputs.dpres = p3_preproc.pseudo_density_dry; //give dry density as input diag_inputs.qv_prev = p3_preproc.qv_prev; auto t_prev = get_field_out("T_prev_micro_step").get_view(); diag_inputs.t_prev = t_prev; @@ -288,6 +300,7 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) // --Diagnostic Outputs diag_outputs.diag_eff_radius_qc = get_field_out("eff_radius_qc").get_view(); diag_outputs.diag_eff_radius_qi = get_field_out("eff_radius_qi").get_view(); + diag_outputs.diag_eff_radius_qr = get_field_out("eff_radius_qr").get_view(); diag_outputs.precip_liq_surf = m_buffer.precip_liq_surf_flux; diag_outputs.precip_ice_surf = m_buffer.precip_ice_surf_flux; @@ -303,10 +316,12 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) history_only.vap_ice_exchange = get_field_out("micro_vap_ice_exchange").get_view(); // -- Set values for the post-amble structure p3_postproc.set_variables(m_num_cols,nk_pack, - prog_state.th,pmid,T_atm,t_prev, + prog_state.th,pmid,pmid_dry,T_atm,t_prev, + pseudo_density,pseudo_density_dry, prog_state.qv, prog_state.qc, prog_state.nc, prog_state.qr,prog_state.nr, prog_state.qi, prog_state.qm, prog_state.ni,prog_state.bm,qv_prev, diag_outputs.diag_eff_radius_qc,diag_outputs.diag_eff_radius_qi, + diag_outputs.diag_eff_radius_qr, diag_outputs.precip_liq_surf,diag_outputs.precip_ice_surf, precip_liq_surf_mass,precip_ice_surf_mass); diff --git a/components/eamxx/src/physics/p3/atmosphere_microphysics.hpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp similarity index 75% rename from components/eamxx/src/physics/p3/atmosphere_microphysics.hpp rename to components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp index 5ec2bddfc3f8..6a9c945e5176 100644 --- a/components/eamxx/src/physics/p3/atmosphere_microphysics.hpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp @@ -70,44 +70,39 @@ class P3Microphysics : public AtmosphereProcess const Spack& pmid_pack(pmid(icol,ipack)); const Spack& T_atm_pack(T_atm(icol,ipack)); const Spack& cld_frac_t_pack(cld_frac_t(icol,ipack)); + const Spack& pseudo_density_pack(pseudo_density(icol,ipack)); + const Spack& pseudo_density_dry_pack(pseudo_density_dry(icol,ipack)); + + //compute dz from full pressure + dz(icol,ipack) = PF::calculate_dz(pseudo_density_pack, pmid_pack, T_atm_pack, qv(icol,ipack)); + /*---------------------------------------------------------------------------------------------------------------------- *Wet to dry mixing ratios: *------------------------- *Since state constituents from the host model (or AD) are wet mixing ratios and P3 needs *these constituents in dry mixing ratios, we convert the wet mixing ratios to dry mixing ratios. - - *NOTE:Function calculate_drymmr_from_wetmmr takes 2 arguments: ( wet mmr and "wet" water vapor mixing ratio) - - *IMPORTANT:Convert "qv wet mmr" to "qv dry mmr" after converting all other constituents to dry mmr as "qv" (as wet mmr) - * is an input for converting all other constituent5Bs to have dry mmr. - *---------------------------------------------------------------------------------------------------------------------- */ - //Since "qv" has a wet mixing ratio, we can use "qv" to compute dry mixing ratios of the following constituents: //Units of all constituents below are [kg/kg(dry-air)] for mass and [#/kg(dry-air)] for number - qc(icol, ipack) = PF::calculate_drymmr_from_wetmmr(qc(icol,ipack),qv(icol,ipack)); //Cloud liquid mass - nc(icol, ipack) = PF::calculate_drymmr_from_wetmmr(nc(icol,ipack),qv(icol,ipack)); //Cloud liquid numbe - qr(icol, ipack) = PF::calculate_drymmr_from_wetmmr(qr(icol,ipack),qv(icol,ipack)); //Rain mass - nr(icol, ipack) = PF::calculate_drymmr_from_wetmmr(nr(icol,ipack),qv(icol,ipack)); //Rain number - qi(icol, ipack) = PF::calculate_drymmr_from_wetmmr(qi(icol,ipack),qv(icol,ipack)); //Cloud ice mass - ni(icol, ipack) = PF::calculate_drymmr_from_wetmmr(ni(icol,ipack),qv(icol,ipack)); //Cloud ice number - qm(icol, ipack) = PF::calculate_drymmr_from_wetmmr(qm(icol,ipack),qv(icol,ipack)); //Rimmed ice mass - bm(icol, ipack) = PF::calculate_drymmr_from_wetmmr(bm(icol,ipack),qv(icol,ipack)); //Rimmed ice number - //Water vapor from previous time step - qv_prev(icol, ipack) = PF::calculate_drymmr_from_wetmmr(qv_prev(icol,ipack),qv(icol,ipack)); + qc(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(qc(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); //Cloud liquid mass + nc(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(nc(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); //Cloud liquid numbe + qr(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(qr(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); //Rain mass + nr(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(nr(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); //Rain number + qi(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(qi(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); //Cloud ice mass + ni(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(ni(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); //Cloud ice number + qm(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(qm(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); //Rimmed ice mass + bm(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(bm(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); //Rimmed ice number + qv(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(qv(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); - // ^^ Ensure that qv is "wet mmr" till this point ^^ - //NOTE: Convert "qv" to dry mmr in the end after converting all other constituents to dry mmr - qv(icol, ipack) = PF::calculate_drymmr_from_wetmmr(qv(icol,ipack),qv(icol,ipack)); + //Water vapor from previous time step + qv_prev(icol, ipack) = PF::calculate_drymmr_from_wetmmr_dp_based(qv_prev(icol,ipack),pseudo_density_pack,pseudo_density_dry_pack); - // Exner + // Exner from full pressure const auto& exner = PF::exner_function(pmid_pack); inv_exner(icol,ipack) = 1.0/exner; - // Potential temperature + // Potential temperature, from full pressure th_atm(icol,ipack) = PF::calculate_theta_from_T(T_atm_pack,pmid_pack); - // DZ - dz(icol,ipack) = PF::calculate_dz(pseudo_density(icol,ipack), pmid_pack, T_atm_pack, qv(icol,ipack)); // Cloud fraction // Set minimum cloud fraction - avoids division by zero cld_frac_l(icol,ipack) = ekat::max(cld_frac_t_pack,mincld); @@ -136,7 +131,9 @@ class P3Microphysics : public AtmosphereProcess int m_ncol, m_npack; Real mincld = 0.0001; // TODO: These should be stored somewhere as more universal constants. Or maybe in the P3 class hpp view_2d_const pmid; + view_2d_const pmid_dry; view_2d_const pseudo_density; + view_2d_const pseudo_density_dry; view_2d T_atm; view_2d_const cld_frac_t; view_2d qv; @@ -157,7 +154,9 @@ class P3Microphysics : public AtmosphereProcess view_2d dz; // Assigning local variables void set_variables(const int ncol, const int npack, - const view_2d_const& pmid_, const view_2d_const& pseudo_density_, const view_2d& T_atm_, + const view_2d_const& pmid_, const view_2d_const& pmid_dry_, + const view_2d_const& pseudo_density_, + const view_2d_const& pseudo_density_dry_, const view_2d& T_atm_, const view_2d_const& cld_frac_t_, const view_2d& qv_, const view_2d& qc_, const view_2d& nc_, const view_2d& qr_, const view_2d& nr_, const view_2d& qi_, const view_2d& qm_, const view_2d& ni_, const view_2d& bm_, const view_2d& qv_prev_, @@ -169,7 +168,9 @@ class P3Microphysics : public AtmosphereProcess m_npack = npack; // IN pmid = pmid_; + pmid_dry = pmid_dry_; pseudo_density = pseudo_density_; + pseudo_density_dry = pseudo_density_dry_; T_atm = T_atm_; cld_frac_t = cld_frac_t_; // OUT @@ -202,8 +203,18 @@ class P3Microphysics : public AtmosphereProcess KOKKOS_INLINE_FUNCTION void operator()(const int icol) const { for (int ipack=0;ipack void Functions ::get_latent_heat(const Int& nj, const Int& nk, view_2d& v, view_2d& s, view_2d& f) { - using ExeSpace = typename KT::ExeSpace; - - constexpr Scalar lapvap = C::LatVap; + constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk); - Kokkos::parallel_for("get_latent_heat", policy, KOKKOS_LAMBDA(const MemberType& team) { - int i = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nk), [&] (const int& k) { - v(i,k) = lapvap; - s(i,k) = lapvap + latice; - f(i,k) = latice; - }); - }); - + Kokkos::deep_copy(v, latvap); + Kokkos::deep_copy(s, latvap + latice); + Kokkos::deep_copy(f, latice); } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp index 9cb626936917..cd8d9ce7ef0e 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp @@ -42,7 +42,7 @@ ::ice_cldliq_wet_growth( Spack dum1{0.}; if (any_if.any()) { - qsat0 = physics::qv_sat( zerodeg,pres, false, context, physics::MurphyKoop, "p3::ice_cldliq_wet_growth" ); + qsat0 = physics::qv_sat_dry( zerodeg,pres, false, context, physics::MurphyKoop, "p3::ice_cldliq_wet_growth" ); qc_growth_rate.set(any_if, ((table_val_qi2qr_melting+table_val_qi2qr_vent_melt*cbrt(sc)*sqrt(rhofaci*rho/mu))* diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp index aadd46721530..dc6c8d16256f 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp @@ -35,7 +35,7 @@ ::ice_melting( if (has_melt_qi.any()) { // Note that qsat0 should be with respect to liquid. Confirmed F90 code did this. - const auto qsat0 = physics::qv_sat(Spack(Tmelt), pres, false, context, physics::MurphyKoop, "p3::ice_melting"); //"false" here means NOT saturation w/ respect to ice. + const auto qsat0 = physics::qv_sat_dry(Spack(Tmelt), pres, false, context, physics::MurphyKoop, "p3::ice_melting"); //"false" here means NOT saturation w/ respect to ice. qi2qr_melt_tend.set(has_melt_qi, ( (table_val_qi2qr_melting+table_val_qi2qr_vent_melt*cbrt(sc)*sqrt(rhofaci*rho/mu)) *((T_atm-Tmelt)*kap-rho*latent_heat_vapor*dv*(qsat0-qv)) diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index f68279af4d0b..22819e64aad1 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -32,6 +32,7 @@ ::p3_main_init( const uview_1d& ze_rain, const uview_1d& diag_eff_radius_qc, const uview_1d& diag_eff_radius_qi, + const uview_1d& diag_eff_radius_qr, const uview_1d& inv_cld_frac_i, const uview_1d& inv_cld_frac_l, const uview_1d& inv_cld_frac_r, @@ -54,6 +55,7 @@ ::p3_main_init( ze_rain(k) = 1.e-22; diag_eff_radius_qc(k) = 10.e-6; diag_eff_radius_qi(k) = 25.e-6; + diag_eff_radius_qr(k) = 500.e-6; inv_cld_frac_i(k) = 1 / cld_frac_i(k); inv_cld_frac_l(k) = 1 / cld_frac_l(k); inv_cld_frac_r(k) = 1 / cld_frac_r(k); @@ -71,7 +73,8 @@ ::p3_main_init( template Int Functions -::p3_main( +::p3_main_internal( + const P3Runtime& runtime_options, const P3PrognosticState& prognostic_state, const P3DiagnosticInputs& diagnostic_inputs, const P3DiagnosticOutputs& diagnostic_outputs, @@ -188,6 +191,7 @@ ::p3_main( const auto oth = ekat::subview(prognostic_state.th, i); const auto odiag_eff_radius_qc = ekat::subview(diagnostic_outputs.diag_eff_radius_qc, i); const auto odiag_eff_radius_qi = ekat::subview(diagnostic_outputs.diag_eff_radius_qi, i); + const auto odiag_eff_radius_qr = ekat::subview(diagnostic_outputs.diag_eff_radius_qr, i); const auto oqv2qi_depos_tend = ekat::subview(diagnostic_outputs.qv2qi_depos_tend, i); const auto orho_qi = ekat::subview(diagnostic_outputs.rho_qi, i); const auto oprecip_liq_flux = ekat::subview(diagnostic_outputs.precip_liq_flux, i); @@ -218,8 +222,8 @@ ::p3_main( p3_main_init( team, nk_pack, ocld_frac_i, ocld_frac_l, ocld_frac_r, oinv_exner, oth, odz, diag_equiv_reflectivity, - ze_ice, ze_rain, odiag_eff_radius_qc, odiag_eff_radius_qi, inv_cld_frac_i, inv_cld_frac_l, - inv_cld_frac_r, exner, T_atm, oqv, inv_dz, + ze_ice, ze_rain, odiag_eff_radius_qc, odiag_eff_radius_qi, odiag_eff_radius_qr, + inv_cld_frac_i, inv_cld_frac_l, inv_cld_frac_r, exner, T_atm, oqv, inv_dz, diagnostic_outputs.precip_liq_surf(i), diagnostic_outputs.precip_ice_surf(i), zero_init); p3_main_part1( @@ -240,7 +244,7 @@ ::p3_main( // main k-loop (for processes): p3_main_part2( - team, nk_pack, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, inv_dt, + team, nk_pack, runtime_options.max_total_ni, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, inv_dt, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, lookup_tables.revap_table_vals, opres, odpres, odz, onc_nuceat_tend, oinv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, oni_activated, oinv_qc_relvar, ocld_frac_i, ocld_frac_l, ocld_frac_r, oqv_prev, ot_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, @@ -296,11 +300,11 @@ ::p3_main( // and compute diagnostic fields for output // p3_main_part3( - team, nk_pack, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, oinv_exner, ocld_frac_l, ocld_frac_r, ocld_frac_i, + team, nk_pack, runtime_options.max_total_ni, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, oinv_exner, ocld_frac_l, ocld_frac_r, ocld_frac_i, rho, inv_rho, rhofaci, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, olatent_heat_vapor, olatent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, ovap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, odiag_eff_radius_qi, diag_diam_qi, - orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc); + orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr); // // merge ice categories with similar properties @@ -327,6 +331,42 @@ ::p3_main( return duration.count(); } +template +Int Functions +::p3_main( + const P3Runtime& runtime_options, + const P3PrognosticState& prognostic_state, + const P3DiagnosticInputs& diagnostic_inputs, + const P3DiagnosticOutputs& diagnostic_outputs, + const P3Infrastructure& infrastructure, + const P3HistoryOnly& history_only, + const P3LookupTables& lookup_tables, + const WorkspaceManager& workspace_mgr, + Int nj, + Int nk) +{ +#ifndef SCREAM_SMALL_KERNELS + return p3_main_internal(runtime_options, + prognostic_state, + diagnostic_inputs, + diagnostic_outputs, + infrastructure, + history_only, + lookup_tables, + workspace_mgr, + nj, nk); +#else + return p3_main_internal_disp(runtime_options, + prognostic_state, + diagnostic_inputs, + diagnostic_outputs, + infrastructure, + history_only, + lookup_tables, + workspace_mgr, + nj, nk); +#endif +} } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp index 929c3f21107c..28be9ab9040b 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp @@ -102,8 +102,8 @@ ::p3_main_part1( rho(k) = dpres(k)/dz(k) / g; inv_rho(k) = 1 / rho(k); - qv_sat_l(k) = physics::qv_sat(T_atm(k), pres(k), false, range_mask, physics::MurphyKoop, "p3::p3_main_part1 (liquid)"); - qv_sat_i(k) = physics::qv_sat(T_atm(k), pres(k), true, range_mask, physics::MurphyKoop, "p3::p3_main_part1 (ice)"); + qv_sat_l(k) = physics::qv_sat_dry(T_atm(k), pres(k), false, range_mask, physics::MurphyKoop, "p3::p3_main_part1 (liquid)"); + qv_sat_i(k) = physics::qv_sat_dry(T_atm(k), pres(k), true, range_mask, physics::MurphyKoop, "p3::p3_main_part1 (ice)"); qv_supersat_i(k) = qv(k) / qv_sat_i(k) - 1; diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp index 0a611d38917b..29fc2fce11d6 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp @@ -21,6 +21,7 @@ void Functions ::p3_main_part2( const MemberType& team, const Int& nk_pack, + const Scalar& max_total_ni, const bool& predictNc, const bool& do_prescribed_CCN, const Scalar& dt, @@ -98,7 +99,6 @@ ::p3_main_part2( constexpr Scalar qsmall = C::QSMALL; constexpr Scalar nsmall = C::NSMALL; constexpr Scalar T_zerodegc = C::T_zerodegc; - constexpr Scalar max_total_ni = C::max_total_ni; constexpr Scalar f1r = C::f1r; constexpr Scalar f2r = C::f2r; constexpr Scalar nmltratio = C::nmltratio; diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp index c45d3190b663..6791248abca4 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp @@ -21,6 +21,7 @@ void Functions ::p3_main_part3( const MemberType& team, const Int& nk_pack, + const Scalar& max_total_ni, const view_dnu_table& dnu, const view_ice_table& ice_table_vals, const uview_1d& inv_exner, @@ -55,11 +56,11 @@ ::p3_main_part3( const uview_1d& diag_diam_qi, const uview_1d& rho_qi, const uview_1d& diag_equiv_reflectivity, - const uview_1d& diag_eff_radius_qc) + const uview_1d& diag_eff_radius_qc, + const uview_1d& diag_eff_radius_qr) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; - constexpr Scalar max_total_ni = C::max_total_ni; constexpr Scalar nsmall = C::NSMALL; Kokkos::parallel_for( @@ -116,6 +117,7 @@ ::p3_main_part3( ze_rain(k).set(qr_gt_small, nr(k)*(mu_r(k)+6)*(mu_r(k)+5)*(mu_r(k)+4)* (mu_r(k)+3)*(mu_r(k)+2)*(mu_r(k)+1)/pow(lamr(k), sp(6.0))); // once f90 is gone, 6 can be int ze_rain(k).set(qr_gt_small, max(ze_rain(k), sp(1.e-22))); + diag_eff_radius_qr(k).set(qr_gt_small, sp(1.5) / lamr(k)); } if (qr_small.any()) { diff --git a/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp index 78d243fe77a2..a9f265d8dcdd 100644 --- a/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp @@ -41,7 +41,7 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& - qr2qv_evap_tend*latent_heat_vapor*inv_cp )*dt); //qv we would have at end of step if we were saturated with respect to liquid - const auto qsl = physics::qv_sat(T_endstep,pres,false,has_sources,physics::MurphyKoop,"p3::prevent_liq_supersaturation"); //"false" means NOT sat w/ respect to ice + const auto qsl = physics::qv_sat_dry(T_endstep,pres,false,has_sources,physics::MurphyKoop,"p3::prevent_liq_supersaturation"); //"false" means NOT sat w/ respect to ice //The balance we seek is: // qv-qv_sinks*dt+qv_sources*frac*dt=qsl+dqsl_dT*(T correction due to conservation) diff --git a/components/eamxx/src/physics/p3/p3_f90.cpp b/components/eamxx/src/physics/p3/p3_f90.cpp index e9e9a677218c..38bb84c416ec 100644 --- a/components/eamxx/src/physics/p3/p3_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_f90.cpp @@ -49,6 +49,7 @@ FortranData::FortranData (Int ncol_, Int nlev_) precip_ice_surf = Array1("precipitation rate, solid m/s", ncol); diag_eff_radius_qc = Array2("effective radius, cloud, m", ncol, nlev); diag_eff_radius_qi = Array2("effective radius, ice, m", ncol, nlev); + diag_eff_radius_qr = Array2("effective radius, rain, m", ncol, nlev); rho_qi = Array2("bulk density of ice, kg/m", ncol, nlev); qv2qi_depos_tend = Array2("qitend due to deposition/sublimation ", ncol, nlev); precip_liq_flux = Array2("grid-box average rain flux (kg m^-2 s^-1), pverp", ncol, nlev+1); @@ -77,7 +78,7 @@ void FortranDataIterator::init (const FortranData::Ptr& dp) { fdipb(dz); fdipb(nc_nuceat_tend); fdipb(nccn_prescribed); fdipb(ni_activated); fdipb(inv_qc_relvar); fdipb(qc); fdipb(nc); fdipb(qr); fdipb(nr); fdipb(qi); fdipb(ni); fdipb(qm); fdipb(bm); fdipb(precip_liq_surf); fdipb(precip_ice_surf); - fdipb(diag_eff_radius_qc); fdipb(diag_eff_radius_qi); fdipb(rho_qi); + fdipb(diag_eff_radius_qc); fdipb(diag_eff_radius_qi); fdipb(diag_eff_radius_qr); fdipb(rho_qi); fdipb(dpres); fdipb(inv_exner); fdipb(qv2qi_depos_tend); fdipb(precip_liq_flux); fdipb(precip_ice_flux); fdipb(cld_frac_r); fdipb(cld_frac_l); fdipb(cld_frac_i); diff --git a/components/eamxx/src/physics/p3/p3_f90.hpp b/components/eamxx/src/physics/p3/p3_f90.hpp index bc5bb75c737a..d07524d9c7a2 100644 --- a/components/eamxx/src/physics/p3/p3_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_f90.hpp @@ -31,7 +31,7 @@ struct FortranData { ni, qm, bm, dpres, inv_exner, qv_prev, t_prev; // Out Array1 precip_liq_surf, precip_ice_surf; - Array2 diag_eff_radius_qc, diag_eff_radius_qi, rho_qi, qv2qi_depos_tend, + Array2 diag_eff_radius_qc, diag_eff_radius_qi, diag_eff_radius_qr, rho_qi, qv2qi_depos_tend, precip_liq_flux, precip_ice_flux, cld_frac_r, cld_frac_l, cld_frac_i; Array3 p3_tend_out; Array2 liq_ice_exchange,vap_liq_exchange,vap_ice_exchange; diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 7b8e3b6d2293..6a8fbc343b04 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -107,6 +107,12 @@ struct Functions using WorkspaceManager = typename ekat::WorkspaceManager; using Workspace = typename WorkspaceManager::Workspace; + // Structure to store p3 runtime options + struct P3Runtime { + // maximum total ice concentration (sum of all categories) (m) + Scalar max_total_ni; + }; + // This struct stores prognostic variables evolved by P3. struct P3PrognosticState { P3PrognosticState() = default; @@ -176,6 +182,8 @@ struct Functions view_2d diag_eff_radius_qc; // Effective ice radius [m] view_2d diag_eff_radius_qi; + // Effective rain radius [m] + view_2d diag_eff_radius_qr; // Bulk density of ice [kg m-3] view_2d rho_qi; // Grid-box average rain flux [kg m^-2 s^-1] pverp @@ -407,6 +415,30 @@ struct Functions const uview_1d& nc_tend, Scalar& precip_liq_surf); +#ifdef SCREAM_SMALL_KERNELS + static void cloud_sedimentation_disp( + const uview_2d& qc_incld, + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& cld_frac_l, + const uview_2d& acn, + const uview_2d& inv_dz, + const view_dnu_table& dnu, + const WorkspaceManager& workspace_mgr, + const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const Scalar& dt, const Scalar& inv_dt, + const bool& do_predict_nc, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& nc_incld, + const uview_2d& mu_c, + const uview_2d& lamc, + const uview_2d& qc_tend, + const uview_2d& nc_tend, + const uview_1d& precip_liq_surf, + const uview_1d& is_nucleat_possible, + const uview_1d& is_hydromet_present); +#endif + // TODO: comment KOKKOS_FUNCTION static void rain_sedimentation( @@ -430,6 +462,30 @@ struct Functions const uview_1d& nr_tend, Scalar& precip_liq_surf); +#ifdef SCREAM_SMALL_KERNELS + static void rain_sedimentation_disp( + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& rhofacr, + const uview_2d& cld_frac_r, + const uview_2d& inv_dz, + const uview_2d& qr_incld, + const WorkspaceManager& workspace_mgr, + const view_2d_table& vn_table_vals, const view_2d_table& vm_table_vals, + const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const Scalar& dt, const Scalar& inv_dt, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& nr_incld, + const uview_2d& mu_r, + const uview_2d& lamr, + const uview_2d& precip_liq_flux, + const uview_2d& qr_tend, + const uview_2d& nr_tend, + const uview_1d& precip_liq_surf, + const uview_1d& is_nucleat_possible, + const uview_1d& is_hydromet_present); +#endif + // TODO: comment KOKKOS_FUNCTION static void ice_sedimentation( @@ -454,6 +510,31 @@ struct Functions const view_ice_table& ice_table_vals, Scalar& precip_ice_surf); +#ifdef SCREAM_SMALL_KERNELS + static void ice_sedimentation_disp( + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& rhofaci, + const uview_2d& cld_frac_i, + const uview_2d& inv_dz, + const WorkspaceManager& workspace_mgr, + const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const Scalar& dt, const Scalar& inv_dt, + const uview_2d& qi, + const uview_2d& qi_incld, + const uview_2d& ni, + const uview_2d& ni_incld, + const uview_2d& qm, + const uview_2d& qm_incld, + const uview_2d& bm, + const uview_2d& bm_incld, + const uview_2d& qi_tend, + const uview_2d& ni_tend, + const view_ice_table& ice_table_vals, + const uview_1d& precip_ice_surf, + const uview_1d& is_nucleat_possible, + const uview_1d& is_hydromet_present); +#endif + // homogeneous freezing of cloud and rain KOKKOS_FUNCTION static void homogeneous_freezing( @@ -472,6 +553,25 @@ struct Functions const uview_1d& bm, const uview_1d& th_atm); +#ifdef SCREAM_SMALL_KERNELS + static void homogeneous_freezing_disp( + const uview_2d& T_atm, + const uview_2d& inv_exner, + const uview_2d& latent_heat_fusion, + const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& qi, + const uview_2d& ni, + const uview_2d& qm, + const uview_2d& bm, + const uview_2d& th_atm, + const uview_1d& is_nucleat_possible, + const uview_1d& is_hydromet_present); +#endif + // -- Find layers // Find the bottom and top of the mixing ratio, e.g., qr. It's worth casing @@ -741,6 +841,12 @@ struct Functions const Int& timestepcount, const bool& force_abort, const Int& source_ind, const MemberType& team, const uview_1d& col_loc); +#ifdef SCREAM_SMALL_KERNELS + static void check_values_disp(const uview_2d& qv, const uview_2d& temp, const Int& ktop, const Int& kbot, + const Int& timestepcount, const bool& force_abort, const Int& source_ind, + const uview_2d& col_loc, const Int& nj, const Int& nk); +#endif + KOKKOS_FUNCTION static void calculate_incloud_mixingratios( const Spack& qc, const Spack& qr, const Spack& qi, const Spack& qm, const Spack& nc, @@ -769,6 +875,7 @@ struct Functions const uview_1d& ze_rain, const uview_1d& diag_eff_radius_qc, const uview_1d& diag_eff_radius_qi, + const uview_1d& diag_eff_radius_qr, const uview_1d& inv_cld_frac_i, const uview_1d& inv_cld_frac_l, const uview_1d& inv_cld_frac_r, @@ -780,6 +887,29 @@ struct Functions Scalar& precip_ice_surf, view_1d_ptr_array& zero_init); +#ifdef SCREAM_SMALL_KERNELS + static void p3_main_init_disp( + const Int& nj,const Int& nk_pack, + const uview_2d& cld_frac_i, const uview_2d& cld_frac_l, + const uview_2d& cld_frac_r, const uview_2d& inv_exner, + const uview_2d& th_atm, const uview_2d& dz, + const uview_2d& diag_equiv_reflectivity, const uview_2d& ze_ice, + const uview_2d& ze_rain, const uview_2d& diag_eff_radius_qc, + const uview_2d& diag_eff_radius_qi, const uview_2d& diag_eff_radius_qr, const uview_2d& inv_cld_frac_i, + const uview_2d& inv_cld_frac_l, const uview_2d& inv_cld_frac_r, + const uview_2d& exner, const uview_2d& T_atm, const uview_2d& qv, + const uview_2d& inv_dz, const uview_1d& precip_liq_surf, const uview_1d& precip_ice_surf, + const uview_2d& mu_r, const uview_2d& lamr, const uview_2d& logn0r, const uview_2d& nu, + const uview_2d& cdist, const uview_2d& cdist1, const uview_2d& cdistr, + const uview_2d& qc_incld, const uview_2d& qr_incld, const uview_2d& qi_incld, + const uview_2d& qm_incld, const uview_2d& nc_incld, const uview_2d& nr_incld, const uview_2d& ni_incld, + const uview_2d& bm_incld, const uview_2d& inv_rho, const uview_2d& prec, const uview_2d& rho, const uview_2d& rhofacr, + const uview_2d& rhofaci, const uview_2d& acn, const uview_2d& qv_sat_l, const uview_2d& qv_sat_i, const uview_2d& sup, + const uview_2d& qv_supersat_i, const uview_2d& qtend_ignore, const uview_2d& ntend_ignore, const uview_2d& mu_c, + const uview_2d& lamc, const uview_2d& rho_qi, const uview_2d& qv2qi_depos_tend, const uview_2d& precip_total_tend, + const uview_2d& nevapr, const uview_2d& precip_liq_flux, const uview_2d& precip_ice_flux); +#endif + KOKKOS_FUNCTION static void p3_main_part1( const MemberType& team, @@ -830,10 +960,62 @@ struct Functions bool& is_nucleat_possible, bool& is_hydromet_present); +#ifdef SCREAM_SMALL_KERNELS + static void p3_main_part1_disp( + const Int& nj, + const Int& nk, + const bool& do_predict_nc, + const bool& do_prescribed_CCN, + const Scalar& dt, + const uview_2d& pres, + const uview_2d& dpres, + const uview_2d& dz, + const uview_2d& nc_nuceat_tend, + const uview_2d& nccn_prescribed, + const uview_2d& inv_exner, + const uview_2d& exner, + const uview_2d& inv_cld_frac_l, + const uview_2d& inv_cld_frac_i, + const uview_2d& inv_cld_frac_r, + const uview_2d& latent_heat_vapor, + const uview_2d& latent_heat_sublim, + const uview_2d& latent_heat_fusion, + const uview_2d& T_atm, + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& qv_sat_l, + const uview_2d& qv_sat_i, + const uview_2d& qv_supersat_i, + const uview_2d& rhofacr, + const uview_2d& rhofaci, + const uview_2d& acn, + const uview_2d& qv, + const uview_2d& th_atm, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& qi, + const uview_2d& ni, + const uview_2d& qm, + const uview_2d& bm, + const uview_2d& qc_incld, + const uview_2d& qr_incld, + const uview_2d& qi_incld, + const uview_2d& qm_incld, + const uview_2d& nc_incld, + const uview_2d& nr_incld, + const uview_2d& ni_incld, + const uview_2d& bm_incld, + const uview_1d& is_nucleat_possible, + const uview_1d& is_hydromet_present); +#endif + KOKKOS_FUNCTION static void p3_main_part2( const MemberType& team, const Int& nk_pack, + const Scalar& max_total_ni, const bool& do_predict_nc, const bool& do_prescribed_CCN, const Scalar& dt, @@ -907,12 +1089,94 @@ struct Functions const uview_1d& pratot, const uview_1d& prctot, bool& is_hydromet_present, - const Int& nk=-1); + const Int& nk); + +#ifdef SCREAM_SMALL_KERNELS + static void p3_main_part2_disp( + const Int& nj, + const Int& nk, + const Scalar& max_total_ni, + const bool& do_predict_nc, + const bool& do_prescribed_CCN, + const Scalar& dt, + const Scalar& inv_dt, + const view_dnu_table& dnu, + const view_ice_table& ice_table_vals, + const view_collect_table& collect_table_vals, + const view_2d_table& revap_table_vals, + const uview_2d& pres, + const uview_2d& dpres, + const uview_2d& dz, + const uview_2d& nc_nuceat_tend, + const uview_2d& inv_exner, + const uview_2d& exner, + const uview_2d& inv_cld_frac_l, + const uview_2d& inv_cld_frac_i, + const uview_2d& inv_cld_frac_r, + const uview_2d& ni_activated, + const uview_2d& inv_qc_relvar, + const uview_2d& cld_frac_i, + const uview_2d& cld_frac_l, + const uview_2d& cld_frac_r, + const uview_2d& qv_prev, + const uview_2d& t_prev, + const uview_2d& T_atm, + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& qv_sat_l, + const uview_2d& qv_sat_i, + const uview_2d& qv_supersat_i, + const uview_2d& rhofacr, + const uview_2d& rhofaci, + const uview_2d& acn, + const uview_2d& qv, + const uview_2d& th_atm, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& qi, + const uview_2d& ni, + const uview_2d& qm, + const uview_2d& bm, + const uview_2d& latent_heat_vapor, + const uview_2d& latent_heat_sublim, + const uview_2d& latent_heat_fusion, + const uview_2d& qc_incld, + const uview_2d& qr_incld, + const uview_2d& qi_incld, + const uview_2d& qm_incld, + const uview_2d& nc_incld, + const uview_2d& nr_incld, + const uview_2d& ni_incld, + const uview_2d& bm_incld, + const uview_2d& mu_c, + const uview_2d& nu, + const uview_2d& lamc, + const uview_2d& cdist, + const uview_2d& cdist1, + const uview_2d& cdistr, + const uview_2d& mu_r, + const uview_2d& lamr, + const uview_2d& logn0r, + const uview_2d& qv2qi_depos_tend, + const uview_2d& precip_total_tend, + const uview_2d& nevapr, + const uview_2d& qr_evap_tend, + const uview_2d& vap_liq_exchange, + const uview_2d& vap_ice_exchange, + const uview_2d& liq_ice_exchange, + const uview_2d& pratot, + const uview_2d& prctot, + const uview_1d& is_nucleat_possible, + const uview_1d& is_hydromet_present); +#endif KOKKOS_FUNCTION static void p3_main_part3( const MemberType& team, const Int& nk_pack, + const Scalar& max_total_ni, const view_dnu_table& dnu, const view_ice_table& ice_table_vals, const uview_1d& inv_exner, @@ -947,10 +1211,82 @@ struct Functions const uview_1d& diag_diam_qi, const uview_1d& rho_qi, const uview_1d& diag_equiv_reflectivity, - const uview_1d& diag_eff_radius_qc); + const uview_1d& diag_eff_radius_qc, + const uview_1d& diag_eff_radius_qr); + +#ifdef SCREAM_SMALL_KERNELS + static void p3_main_part3_disp( + const Int& nj, + const Int& nk_pack, + const Scalar& max_total_ni, + const view_dnu_table& dnu, + const view_ice_table& ice_table_vals, + const uview_2d& inv_exner, + const uview_2d& cld_frac_l, + const uview_2d& cld_frac_r, + const uview_2d& cld_frac_i, + const uview_2d& rho, + const uview_2d& inv_rho, + const uview_2d& rhofaci, + const uview_2d& qv, + const uview_2d& th_atm, + const uview_2d& qc, + const uview_2d& nc, + const uview_2d& qr, + const uview_2d& nr, + const uview_2d& qi, + const uview_2d& ni, + const uview_2d& qm, + const uview_2d& bm, + const uview_2d& latent_heat_vapor, + const uview_2d& latent_heat_sublim, + const uview_2d& mu_c, + const uview_2d& nu, + const uview_2d& lamc, + const uview_2d& mu_r, + const uview_2d& lamr, + const uview_2d& vap_liq_exchange, + const uview_2d& ze_rain, + const uview_2d& ze_ice, + const uview_2d& diag_vm_qi, + const uview_2d& diag_eff_radius_qi, + const uview_2d& diag_diam_qi, + const uview_2d& rho_qi, + const uview_2d& diag_equiv_reflectivity, + const uview_2d& diag_eff_radius_qc, + const uview_2d& diag_eff_radius_qr, + const uview_1d& is_nucleat_possible, + const uview_1d& is_hydromet_present); +#endif // Return microseconds elapsed static Int p3_main( + const P3Runtime& runtime_options, + const P3PrognosticState& prognostic_state, + const P3DiagnosticInputs& diagnostic_inputs, + const P3DiagnosticOutputs& diagnostic_outputs, + const P3Infrastructure& infrastructure, + const P3HistoryOnly& history_only, + const P3LookupTables& lookup_tables, + const WorkspaceManager& workspace_mgr, + Int nj, // number of columns + Int nk); // number of vertical cells per column + + static Int p3_main_internal( + const P3Runtime& runtime_options, + const P3PrognosticState& prognostic_state, + const P3DiagnosticInputs& diagnostic_inputs, + const P3DiagnosticOutputs& diagnostic_outputs, + const P3Infrastructure& infrastructure, + const P3HistoryOnly& history_only, + const P3LookupTables& lookup_tables, + const WorkspaceManager& workspace_mgr, + Int nj, // number of columns + Int nk); // number of vertical cells per column + +#ifdef SCREAM_SMALL_KERNELS + static Int p3_main_internal_disp( + const P3Runtime& runtime_options, const P3PrognosticState& prognostic_state, const P3DiagnosticInputs& diagnostic_inputs, const P3DiagnosticOutputs& diagnostic_outputs, @@ -960,6 +1296,7 @@ struct Functions const WorkspaceManager& workspace_mgr, Int nj, // number of columns Int nk); // number of vertical cells per column +#endif KOKKOS_FUNCTION static void ice_supersat_conservation(Spack& qidep, Spack& qinuc, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& latent_heat_sublim, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context = Smask(true)); @@ -1035,5 +1372,5 @@ void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, # include "p3_nr_conservation_impl.hpp" # include "p3_ni_conservation_impl.hpp" # include "p3_prevent_liq_supersaturation_impl.hpp" -#endif // GPU || !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE +#endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE #endif // P3_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 11ef4c4085a7..1f6a9d26d871 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -218,14 +218,14 @@ void p3_main_part3_c( Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, - Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc); + Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); void p3_main_c( Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, Real* precip_ice_surf, Int its, Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, - Real* diag_eff_radius_qi, Real* rho_qi, bool do_predict_nc, bool do_prescribed, Real* dpres, Real* inv_exner, + Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, Real* rho_qi, bool do_predict_nc, bool do_prescribed, Real* dpres, Real* inv_exner, Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev, Real* elapsed_s); @@ -782,7 +782,8 @@ P3MainPart3Data::P3MainPart3Data( &qv, &th_atm, &qc, &nc, &qr, &nr, &qi, &ni, &qm, &bm, &latent_heat_vapor, &latent_heat_sublim, &mu_c, &nu, &lamc, &mu_r, &lamr, &vap_liq_exchange, - &ze_rain, &ze_ice, &diag_vm_qi, &diag_eff_radius_qi, &diag_diam_qi, &rho_qi, &diag_equiv_reflectivity, &diag_eff_radius_qc} }), + &ze_rain, &ze_ice, &diag_vm_qi, &diag_eff_radius_qi, &diag_diam_qi, &rho_qi, &diag_equiv_reflectivity, + &diag_eff_radius_qc, &diag_eff_radius_qr} }), kts(kts_), kte(kte_), kbot(kbot_), ktop(ktop_), kdir(kdir_) {} @@ -794,7 +795,7 @@ void p3_main_part3(P3MainPart3Data& d) d.inv_exner, d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, d.rho, d.inv_rho, d.rhofaci, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.latent_heat_vapor, d.latent_heat_sublim, d.mu_c, d.nu, d.lamc, d.mu_r, d.lamr, d.vap_liq_exchange, - d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc); + d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc, d.diag_eff_radius_qr); } /////////////////////////////////////////////////////////////////////////////// @@ -804,7 +805,7 @@ P3MainData::P3MainData( PhysicsTestData( { {(ite_ - its_) + 1, (kte_ - kts_) + 1}, {(ite_ - its_) + 1, (kte_ - kts_) + 2} }, { { &pres, &dz, &nc_nuceat_tend, &nccn_prescribed, &ni_activated, &dpres, &inv_exner, &cld_frac_i, &cld_frac_l, &cld_frac_r, &inv_qc_relvar, &qc, &nc, &qr, &nr, &qi, &qm, &ni, &bm, &qv, &th_atm, &qv_prev, &t_prev, - &diag_eff_radius_qc, &diag_eff_radius_qi, &rho_qi, &mu_c, &lamc, &qv2qi_depos_tend, &precip_total_tend, &nevapr, + &diag_eff_radius_qc, &diag_eff_radius_qi, &diag_eff_radius_qr, &rho_qi, &mu_c, &lamc, &qv2qi_depos_tend, &precip_total_tend, &nevapr, &qr_evap_tend, &liq_ice_exchange, &vap_liq_exchange, &vap_ice_exchange, &precip_liq_flux, &precip_ice_flux}, {&precip_liq_surf, &precip_ice_surf} }), // these two are (ni, nk+1) @@ -819,7 +820,7 @@ void p3_main(P3MainData& d) p3_main_c( d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, - d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, + d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, d.diag_eff_radius_qr, d.rho_qi, d.do_predict_nc, d.do_prescribed_CCN, d.dpres, d.inv_exner, d.qv2qi_depos_tend, d.precip_liq_flux, d.precip_ice_flux, d.cld_frac_r, d.cld_frac_l, d.cld_frac_i, d.liq_ice_exchange, d.vap_liq_exchange, d.vap_ice_exchange, d.qv_prev, d.t_prev, &d.elapsed_s); @@ -1684,6 +1685,7 @@ void p3_main_part2_f( const Int nk = (kte - kts) + 1; const Int nk_pack = ekat::npack(nk); + const Real max_total_ni = 740.0e3; // Hard-code this value for F90 comparison // Set up views std::vector temp_d(P3MainPart2Data::NUM_ARRAYS); @@ -1773,7 +1775,7 @@ void p3_main_part2_f( Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { P3F::p3_main_part2( - team, nk_pack, do_predict_nc, do_prescribed_CCN, dt, inv_dt, dnu, ice_table_vals, collect_table_vals, revap_table_vals, + team, nk_pack, max_total_ni, do_predict_nc, do_prescribed_CCN, dt, inv_dt, dnu, ice_table_vals, collect_table_vals, revap_table_vals, pres_d, dpres_d, dz_d, nc_nuceat_tend_d, inv_exner_d, exner_d, inv_cld_frac_l_d, inv_cld_frac_i_d, inv_cld_frac_r_d, ni_activated_d, inv_qc_relvar_d, cld_frac_i_d, cld_frac_l_d, cld_frac_r_d, qv_prev_d, t_prev_d, t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, @@ -1819,7 +1821,7 @@ void p3_main_part3_f( Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, - Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc) + Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr) { using P3F = Functions; @@ -1839,6 +1841,7 @@ void p3_main_part3_f( const Int nk = (kte - kts) + 1; const Int nk_pack = ekat::npack(nk); + const Real max_total_ni = 740.0e3; // Hard-code this value for F90 comparison // Set up views std::vector temp_d(P3MainPart3Data::NUM_ARRAYS); @@ -1847,7 +1850,7 @@ void p3_main_part3_f( inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, - rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc}, + rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr}, nk, temp_d); view_1d @@ -1883,7 +1886,8 @@ void p3_main_part3_f( diag_diam_qi_d (temp_d[29]), rho_qi_d (temp_d[30]), diag_equiv_reflectivity_d (temp_d[31]), - diag_eff_radius_qc_d (temp_d[32]); + diag_eff_radius_qc_d (temp_d[32]), + diag_eff_radius_qr_d (temp_d[33]); // Call core function from kernel const auto dnu = P3GlobalForFortran::dnu(); @@ -1891,14 +1895,14 @@ void p3_main_part3_f( auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - P3F::p3_main_part3(team, nk_pack, dnu, ice_table_vals, + P3F::p3_main_part3(team, nk_pack, max_total_ni, dnu, ice_table_vals, inv_exner_d, cld_frac_l_d, cld_frac_r_d, cld_frac_i_d, rho_d, inv_rho_d, rhofaci_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, latent_heat_vapor_d, latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, - diag_equiv_reflectivity_d, diag_eff_radius_qc_d); + diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d); }); // Sync back to host @@ -1906,13 +1910,14 @@ void p3_main_part3_f( rho_d, inv_rho_d, rhofaci_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, latent_heat_vapor_d, latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, - diag_diam_qi_d, rho_qi_d, diag_equiv_reflectivity_d, diag_eff_radius_qc_d + diag_diam_qi_d, rho_qi_d, diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d }; ekat::device_to_host({ rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, - diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc + diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, + diag_eff_radius_qr }, nk, inout_views); } @@ -1922,7 +1927,7 @@ Int p3_main_f( Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, Real* precip_ice_surf, Int its, Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, - Real* diag_eff_radius_qi, Real* rho_qi, bool do_predict_nc, bool do_prescribed_CCN, Real* dpres, Real* inv_exner, + Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, Real* rho_qi, bool do_predict_nc, bool do_prescribed_CCN, Real* dpres, Real* inv_exner, Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev) { @@ -1958,7 +1963,7 @@ Int p3_main_f( std::vector dim2_sizes(P3MainData::NUM_ARRAYS, nk); std::vector ptr_array = { pres, dz, nc_nuceat_tend, nccn_prescribed, ni_activated, dpres, inv_exner, cld_frac_i, cld_frac_l, cld_frac_r, inv_qc_relvar, - qc, nc, qr, nr, qi, qm, ni, bm, qv, th_atm, qv_prev, t_prev, diag_eff_radius_qc, diag_eff_radius_qi, + qc, nc, qr, nr, qi, qm, ni, bm, qv, th_atm, qv_prev, t_prev, diag_eff_radius_qc, diag_eff_radius_qi, diag_eff_radius_qr, rho_qi, qv2qi_depos_tend, liq_ice_exchange, vap_liq_exchange, vap_ice_exchange, precip_liq_flux, precip_ice_flux, precip_liq_surf, precip_ice_surf }; @@ -2005,15 +2010,16 @@ Int p3_main_f( t_prev_d (temp_d[counter++]), diag_eff_radius_qc_d (temp_d[counter++]), diag_eff_radius_qi_d (temp_d[counter++]), - rho_qi_d (temp_d[counter++]), //25 + diag_eff_radius_qr_d (temp_d[counter++]), //25 + rho_qi_d (temp_d[counter++]), qv2qi_depos_tend_d (temp_d[counter++]), liq_ice_exchange_d (temp_d[counter++]), vap_liq_exchange_d (temp_d[counter++]), - vap_ice_exchange_d (temp_d[counter++]), - precip_liq_flux_d (temp_d[counter++]), //30 + vap_ice_exchange_d (temp_d[counter++]), //30 + precip_liq_flux_d (temp_d[counter++]), precip_ice_flux_d (temp_d[counter++]), precip_liq_surf_temp_d (temp_d[counter++]), - precip_ice_surf_temp_d (temp_d[counter++]); //33 + precip_ice_surf_temp_d (temp_d[counter++]); //34 // Special cases: precip_liq_surf=1d(ni), precip_ice_surf=1d(ni), col_location=2d(nj, 3) sview_1d precip_liq_surf_d("precip_liq_surf_d", nj), precip_ice_surf_d("precip_ice_surf_d", nj); @@ -2041,7 +2047,7 @@ Int p3_main_f( cld_frac_l_d, cld_frac_r_d, pres_d, dz_d, dpres_d, inv_exner_d, qv_prev_d, t_prev_d}; P3F::P3DiagnosticOutputs diag_outputs{qv2qi_depos_tend_d, precip_liq_surf_d, - precip_ice_surf_d, diag_eff_radius_qc_d, diag_eff_radius_qi_d, + precip_ice_surf_d, diag_eff_radius_qc_d, diag_eff_radius_qi_d, diag_eff_radius_qr_d, rho_qi_d,precip_liq_flux_d, precip_ice_flux_d}; P3F::P3Infrastructure infrastructure{dt, it, its, ite, kts, kte, do_predict_nc, do_prescribed_CCN, col_location_d}; @@ -2059,13 +2065,14 @@ Int p3_main_f( P3F::P3LookupTables lookup_tables{mu_r_table_vals, vn_table_vals, vm_table_vals, revap_table_vals, ice_table_vals, collect_table_vals, dnu_table_vals}; + P3F::P3Runtime runtime_options{740.0e3}; // Create local workspace const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); ekat::WorkspaceManager workspace_mgr(nk_pack, 52, policy); - auto elapsed_microsec = P3F::p3_main(prog_state, diag_inputs, diag_outputs, infrastructure, + auto elapsed_microsec = P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, history_only, lookup_tables, workspace_mgr, nj, nk); Kokkos::parallel_for(nj, KOKKOS_LAMBDA(const Int& i) { @@ -2076,7 +2083,7 @@ Int p3_main_f( // Sync back to host std::vector inout_views = { qc_d, nc_d, qr_d, nr_d, qi_d, qm_d, ni_d, bm_d, qv_d, th_atm_d, - diag_eff_radius_qc_d, diag_eff_radius_qi_d, rho_qi_d, + diag_eff_radius_qc_d, diag_eff_radius_qi_d, diag_eff_radius_qr_d, rho_qi_d, qv2qi_depos_tend_d, liq_ice_exchange_d, vap_liq_exchange_d, vap_ice_exchange_d, precip_liq_flux_d, precip_ice_flux_d, precip_liq_surf_temp_d, precip_ice_surf_temp_d @@ -2090,7 +2097,7 @@ Int p3_main_f( dim1_sizes_out[dim_sizes_out_len-1] = 1; dim2_sizes_out[dim_sizes_out_len-1] = nj; // precip_ice_surf ekat::device_to_host({ - qc, nc, qr, nr, qi, qm, ni, bm, qv, th_atm, diag_eff_radius_qc, diag_eff_radius_qi, + qc, nc, qr, nr, qi, qm, ni, bm, qv, th_atm, diag_eff_radius_qc, diag_eff_radius_qi, diag_eff_radius_qr, rho_qi, qv2qi_depos_tend, liq_ice_exchange, vap_liq_exchange, vap_ice_exchange, precip_liq_flux, precip_ice_flux, precip_liq_surf, precip_ice_surf }, diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index f885c731b044..926c9ab47017 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -723,7 +723,7 @@ struct P3MainPart2Data : public PhysicsTestData struct P3MainPart3Data : public PhysicsTestData { - static constexpr size_t NUM_ARRAYS = 33; + static constexpr size_t NUM_ARRAYS = 34; // Inputs Int kts, kte, kbot, ktop, kdir; @@ -734,7 +734,8 @@ struct P3MainPart3Data : public PhysicsTestData *qv, *th_atm, *qc, *nc, *qr, *nr, *qi, *ni, *qm, *bm, *latent_heat_vapor, *latent_heat_sublim, *mu_c, *nu, *lamc, *mu_r, *lamr, *vap_liq_exchange, - *ze_rain, *ze_ice, *diag_vm_qi, *diag_eff_radius_qi, *diag_diam_qi, *rho_qi, *diag_equiv_reflectivity, *diag_eff_radius_qc; + *ze_rain, *ze_ice, *diag_vm_qi, *diag_eff_radius_qi, *diag_diam_qi, *rho_qi, + *diag_equiv_reflectivity, *diag_eff_radius_qc, *diag_eff_radius_qr; P3MainPart3Data(Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_); @@ -747,7 +748,7 @@ struct P3MainPart3Data : public PhysicsTestData struct P3MainData : public PhysicsTestData { - static constexpr size_t NUM_ARRAYS = 34; + static constexpr size_t NUM_ARRAYS = 35; static constexpr size_t NUM_INPUT_ARRAYS = 24; // Inputs @@ -760,7 +761,7 @@ struct P3MainData : public PhysicsTestData Real* qc, *nc, *qr, *nr, *qi, *qm, *ni, *bm, *qv, *th_atm; // Out - Real *diag_eff_radius_qc, *diag_eff_radius_qi, *rho_qi, *mu_c, *lamc, *qv2qi_depos_tend, *precip_total_tend, *nevapr, + Real *diag_eff_radius_qc, *diag_eff_radius_qi, *diag_eff_radius_qr, *rho_qi, *mu_c, *lamc, *qv2qi_depos_tend, *precip_total_tend, *nevapr, *qr_evap_tend, *liq_ice_exchange, *vap_liq_exchange, *vap_ice_exchange, *precip_liq_flux, *precip_ice_flux, *precip_liq_surf, *precip_ice_surf; Real elapsed_s; @@ -940,14 +941,14 @@ void p3_main_part3_f( Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, - Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc); + Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); Int p3_main_f( Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, Real* precip_ice_surf, Int its, Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, - Real* diag_eff_radius_qi, Real* rho_qi, bool do_predict_nc, bool do_prescribed_CCN, Real* dpres, Real* inv_exner, + Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, Real* rho_qi, bool do_predict_nc, bool do_prescribed_CCN, Real* dpres, Real* inv_exner, Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev); diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index 1b2986c0e715..d75af1cb2edb 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -124,7 +124,7 @@ end subroutine p3_init_c subroutine p3_main_c(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend, & + diag_eff_radius_qi,diag_eff_radius_qr,rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend, & precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i,liq_ice_exchange, & vap_liq_exchange, vap_ice_exchange, qv_prev, t_prev, elapsed_s) bind(C) use micro_p3, only : p3_main @@ -137,7 +137,9 @@ subroutine p3_main_c(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & real(kind=c_real), value, intent(in) :: dt real(kind=c_real), intent(out), dimension(its:ite) :: precip_liq_surf, precip_ice_surf real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qc - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qi, rho_qi + real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qi + real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qr + real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: rho_qi integer(kind=c_int), value, intent(in) :: its,ite, kts,kte, it logical(kind=c_bool), value, intent(in) :: do_predict_nc,do_prescribed_CCN @@ -168,7 +170,7 @@ subroutine p3_main_c(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & call p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend,precip_total_tend,nevapr, & + diag_eff_radius_qi,diag_eff_radius_qr, rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend,precip_total_tend,nevapr, & qr_evap_tend,precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i,p3_tend_out,mu_c,lamc,liq_ice_exchange,& vap_liq_exchange,vap_ice_exchange,qv_prev,t_prev,col_location,elapsed_s) end subroutine p3_main_c @@ -913,7 +915,7 @@ subroutine p3_main_part3_c(kts, kte, kbot, ktop, kdir, & inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc) bind(C) + ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr) bind(C) use micro_p3, only: p3_main_part3 @@ -925,13 +927,14 @@ subroutine p3_main_part3_c(kts, kte, kbot, ktop, kdir, & qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & mu_c, nu, lamc, mu_r, & lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc + ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, & + diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr call p3_main_part3(kts, kte, kbot, ktop, kdir, & inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc) + ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr) end subroutine p3_main_part3_c diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/p3_main_wrap.cpp index f7fdf2186df5..ed46e281a675 100644 --- a/components/eamxx/src/physics/p3/p3_main_wrap.cpp +++ b/components/eamxx/src/physics/p3/p3_main_wrap.cpp @@ -14,7 +14,7 @@ extern "C" { Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, Real* precip_ice_surf, Int its, - Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, Real* diag_eff_radius_qi, + Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, Real* rho_qi, bool do_predict_nc, bool do_prescribed_CCN, Real* dpres, Real* inv_exner, Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, // 1 extra column size @@ -35,7 +35,7 @@ Int p3_main_wrap(const FortranData& d, bool use_fortran) { d.qm.data(), d.ni.data(), d.bm.data(), d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), d.ni_activated.data(), d.inv_qc_relvar.data(), d.it, d.precip_liq_surf.data(), d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, - d.diag_eff_radius_qc.data(), d.diag_eff_radius_qi.data(), d.rho_qi.data(), + d.diag_eff_radius_qc.data(), d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), d.precip_liq_flux.data(), d.precip_ice_flux.data(), d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), d.liq_ice_exchange.data(), d.vap_liq_exchange.data(),d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data(), &elapsed_s); @@ -47,7 +47,7 @@ Int p3_main_wrap(const FortranData& d, bool use_fortran) { d.bm.data(), d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), d.ni_activated.data(), d.inv_qc_relvar.data(), d.it, d.precip_liq_surf.data(), d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, d.diag_eff_radius_qc.data(), - d.diag_eff_radius_qi.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, + d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), d.precip_liq_flux.data(), d.precip_ice_flux.data(), d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 8a46639bc419..fd864e427815 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -1,6 +1,5 @@ include(ScreamUtils) -set(NEED_LIBS p3 physics_share scream_share) set(P3_TESTS_SRCS p3_tests.cpp p3_unit_tests.cpp @@ -48,42 +47,61 @@ endif() # NOTE: tests inside this if statement won't be built in a baselines-only build if (NOT SCREAM_BASELINES_ONLY) - CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" "${NEED_LIBS}" - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics") + CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" + LIBS p3 + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3;physics") # Make sure that a diff in the two implementation triggers a failed test (in debug only) - CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp "${NEED_LIBS}" - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS} - LABELS "p3;physics;fail") + CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp + LIBS p3 + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3;physics;fail" + PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) + + if (NOT SCREAM_SMALL_KERNELS) + CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" + LIBS p3_sk + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3_sk;physics") + + # Make sure that a diff in the two implementation triggers a failed test (in debug only) + CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp + LIBS p3_sk + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3_sk;physics;fail" + PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) + endif() endif() if (SCREAM_ENABLE_BASELINE_TESTS) set(BASELINE_FILE_ARG "-b ${SCREAM_TEST_DATA_DIR}/p3_run_and_cmp.baseline") - CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" "${NEED_LIBS}" - EXCLUDE_MAIN_CPP) + CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" + LIBS p3 + EXCLUDE_MAIN_CPP) CreateUnitTestFromExec(p3_run_and_cmp_cxx p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics") + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics") CreateUnitTestFromExec(p3_run_and_cmp_f90 p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f ${BASELINE_FILE_ARG}" - LABELS "p3;physics") + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "-f ${BASELINE_FILE_ARG}" + LABELS "p3;physics") # Make sure that a diff from baselines triggers a failed test (in debug only) - CreateUnitTest(p3_run_and_cmp_cxx_fail "p3_run_and_cmp.cpp" "${NEED_LIBS}" - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS} - EXCLUDE_MAIN_CPP - LABELS "p3;physics;fail") + CreateUnitTest(p3_run_and_cmp_cxx_fail "p3_run_and_cmp.cpp" + LIBS p3 + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics;fail" + EXCLUDE_MAIN_CPP + PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) # # Use fake tests to generate shell commands to generate baselines diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index 61cd30ba499e..bad0ca7014da 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -296,7 +296,7 @@ static void run_bfb_p3_main_part3() d.inv_exner, d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, d.rho, d.inv_rho, d.rhofaci, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.latent_heat_vapor, d.latent_heat_sublim, d.mu_c, d.nu, d.lamc, d.mu_r, d.lamr, d.vap_liq_exchange, - d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc); + d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc, d.diag_eff_radius_qr); } if (SCREAM_BFB_TESTING) { @@ -333,6 +333,7 @@ static void run_bfb_p3_main_part3() REQUIRE(isds_fortran[i].rho_qi[k] == isds_cxx[i].rho_qi[k]); REQUIRE(isds_fortran[i].diag_equiv_reflectivity[k] == isds_cxx[i].diag_equiv_reflectivity[k]); REQUIRE(isds_fortran[i].diag_eff_radius_qc[k] == isds_cxx[i].diag_eff_radius_qc[k]); + REQUIRE(isds_fortran[i].diag_eff_radius_qr[k] == isds_cxx[i].diag_eff_radius_qr[k]); } } } @@ -392,15 +393,15 @@ static void run_bfb_p3_main() // Get data from cxx for (auto& d : isds_cxx) { - d.transpose(); + d.template transpose(); p3_main_f( d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, - d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, + d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, d.diag_eff_radius_qr, d.rho_qi, d.do_predict_nc, d.do_prescribed_CCN, d.dpres, d.inv_exner, d.qv2qi_depos_tend, d.precip_liq_flux, d.precip_ice_flux, d.cld_frac_r, d.cld_frac_l, d.cld_frac_i, d.liq_ice_exchange, d.vap_liq_exchange, d.vap_ice_exchange, d.qv_prev, d.t_prev); - d.transpose(); + d.template transpose(); } if (SCREAM_BFB_TESTING) { @@ -421,6 +422,7 @@ static void run_bfb_p3_main() REQUIRE(df90.th_atm[t] == dcxx.th_atm[t]); REQUIRE(df90.diag_eff_radius_qc[t] == dcxx.diag_eff_radius_qc[t]); REQUIRE(df90.diag_eff_radius_qi[t] == dcxx.diag_eff_radius_qi[t]); + REQUIRE(df90.diag_eff_radius_qr[t] == dcxx.diag_eff_radius_qr[t]); REQUIRE(df90.rho_qi[t] == dcxx.rho_qi[t]); REQUIRE(df90.mu_c[t] == dcxx.mu_c[t]); REQUIRE(df90.lamc[t] == dcxx.lamc[t]); diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index 4d3f743e7aa0..55501a30aa33 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -52,7 +52,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Spack qv_endstep=qv - qv_sinks*dt + qv_sources*dt; Spack T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*latent_heat_sublim*inv_cp - qr2qv_evap_tend_tmp*latent_heat_vapor*inv_cp )*dt; - Spack qsl = physics::qv_sat(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice + Spack qsl = physics::qv_sat_dry(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice //just require index 0 since all entries are identical REQUIRE( qv[0]::TestPreventLiqSupersaturation { qv_endstep=qv - qv_sinks*dt + qv_sources*dt; T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*latent_heat_sublim*inv_cp - qr2qv_evap_tend_tmp*latent_heat_vapor*inv_cp )*dt; - qsl = physics::qv_sat(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice + qsl = physics::qv_sat_dry(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice //just require index 0 since all entries are identical REQUIRE(qv_endstep[0]<=qsl[0]); diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 587912e7dde2..8a1fa4969bdf 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -127,12 +127,20 @@ static void run_bfb_rain_sed() { auto engine = setup_random_test(); + // F90 is quite slow on weaver, so we decrease dt to reduce + // the number of steps in rain_sed. +#ifdef EAMXX_ENABLE_GPU + constexpr Scalar dt = 5.800E+01; +#else + constexpr Scalar dt = 1.800E+03; +#endif + RainSedData rsds_fortran[] = { - // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_liq_surf - RainSedData(1, 72, 27, 72, -1, 1.800E+03, 5.556E-04, 0.0), - RainSedData(1, 72, 72, 27, 1, 1.800E+03, 5.556E-04, 1.0), - RainSedData(1, 72, 27, 27, -1, 1.800E+03, 5.556E-04, 0.0), - RainSedData(1, 72, 27, 27, 1, 1.800E+03, 5.556E-04, 2.0), + // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_liq_surf + RainSedData(1, 72, 27, 72, -1, dt, 1/dt, 0.0), + RainSedData(1, 72, 72, 27, 1, dt, 1/dt, 1.0), + RainSedData(1, 72, 27, 27, -1, dt, 1/dt, 0.0), + RainSedData(1, 72, 27, 27, 1, dt, 1/dt, 2.0), }; static constexpr Int num_runs = sizeof(rsds_fortran) / sizeof(RainSedData); diff --git a/components/eamxx/src/physics/p3/tests/p3_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_tests.cpp index 8e1a636d300e..d1a64f48520c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_tests.cpp @@ -14,7 +14,7 @@ TEST_CASE("FortranDataIterator", "p3") { using scream::p3::ic::Factory; const auto d = Factory::create(Factory::mixed); scream::p3::FortranDataIterator fdi(d); - REQUIRE(fdi.nfield() == 34); + REQUIRE(fdi.nfield() == 35); const auto& f = fdi.getfield(0); REQUIRE(f.dim == 2); REQUIRE(f.extent[0] == 1); diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index 0f9f6cda545b..df5b80cdc19e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -996,7 +996,7 @@ template struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi { static void impose_max_total_ni_bfb_test(){ - constexpr Scalar max_total_ni = C::max_total_ni; + constexpr Scalar max_total_ni = 740.0e3; ImposeMaxTotalNiData dc[max_pack_size]= { // ni_local, max_total_ni, inv_rho_local diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 6a11cb1f12ec..329b7022756c 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -3,29 +3,78 @@ #include "share/atm_process/atmosphere_process.hpp" -// TODO: in the future, you may want to guard these headers, -// in case we add more options for each parametrization, -// and we want to only register the ones built, -// without hardcoding all of them. - -#include "physics/p3/atmosphere_microphysics.hpp" -#include "physics/shoc/atmosphere_macrophysics.hpp" -#include "physics/cld_fraction/atmosphere_cld_fraction.hpp" -#include "physics/rrtmgp/atmosphere_radiation.hpp" -#include "physics/spa/atmosphere_prescribed_aerosol.hpp" -#include "physics/nudging/atmosphere_nudging.hpp" +// Only include headers and register processes for libs that have been linked in + +#ifdef EAMXX_HAS_P3 +#include "physics/p3/eamxx_p3_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_SHOC +#include "physics/shoc/eamxx_shoc_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_CLD_FRACTION +#include "physics/cld_fraction/eamxx_cld_fraction_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_RRTMGP +#include "physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_SPA +#include "physics/spa/eamxx_spa_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_NUDGING +#include "physics/nudging/eamxx_nudging_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_MAM +#include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" +#include "physics/mam/eamxx_mam_optics_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_COSP +#include "physics/cosp/eamxx_cosp.hpp" +#endif +#ifdef EAMXX_HAS_TMS +#include "physics/tms/eamxx_tms_process_interface.hpp" +#endif +#ifdef EAMXX_HAS_ML_CORRECTION +#include "physics/ml_correction/eamxx_ml_correction_process_interface.hpp" +#endif namespace scream { inline void register_physics () { auto& proc_factory = AtmosphereProcessFactory::instance(); - +#ifdef EAMXX_HAS_P3 proc_factory.register_product("p3",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_SHOC proc_factory.register_product("SHOC",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_CLD_FRACTION proc_factory.register_product("CldFraction",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_RRTMGP proc_factory.register_product("RRTMGP",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_SPA proc_factory.register_product("SPA",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_NUDGING proc_factory.register_product("Nudging",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_MAM + proc_factory.register_product("mam4_micro",&create_atmosphere_process); + proc_factory.register_product("mam4_optics",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_COSP + proc_factory.register_product("Cosp",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_TMS + proc_factory.register_product("tms",&create_atmosphere_process); +#endif +#ifdef EAMXX_HAS_ML_CORRECTION + proc_factory.register_product("MLCorrection",&create_atmosphere_process); +#endif + + // If no physics was enabled, silence compile warning about unused var + (void) proc_factory; } } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index e531a249fcb4..cc4af5506d4a 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -8,6 +8,7 @@ include(ScreamUtils) # RRTMGP++ requires YAKL +string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci) if (TARGET yakl) # Other E3SM components are building YAKL... message ("It appears some other part of E3SM is building YAKL.\n" @@ -39,16 +40,18 @@ else () EkatDisableAllWarning(yakl) # For debug builds, set -DYAKL_DEBUG - string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci) if (CMAKE_BUILD_TYPE_ci STREQUAL "debug") target_compile_definitions(yakl PUBLIC YAKL_DEBUG) endif() endif() -# Whether we built yakl or it was already built, we still need to ensure -# debug builds have --fmad=false in it. The EKAT util automatically sets -# the flag for debug builds -SetCudaFlags(yakl CUDA_LANG) +# See eamxx/src/dynamics/homme/CMakeLists.txt for an explanation of this +# workaround. +if ((SCREAM_MACHINE STREQUAL "ascent" OR SCREAM_MACHINE STREQUAL "pm-gpu") AND CMAKE_BUILD_TYPE_ci STREQUAL "debug") + SetCudaFlags(yakl CUDA_LANG FLAGS -UNDEBUG) +else() + SetCudaFlags(yakl CUDA_LANG) +endif() ################################## # RRTMGP # @@ -71,6 +74,7 @@ set(EXTERNAL_SRC ${EAM_RRTMGP_DIR}/external/cpp/extensions/fluxes_byband/mo_fluxes_byband_kernels.cpp ) add_library(rrtmgp ${EXTERNAL_SRC}) +target_compile_definitions(rrtmgp PUBLIC EAMXX_HAS_RRTMGP) EkatDisableAllWarning(rrtmgp) yakl_process_target(rrtmgp) @@ -133,7 +137,7 @@ target_include_directories(scream_rrtmgp_yakl SYSTEM PUBLIC ################################## set(SCREAM_RRTMGP_SOURCES - atmosphere_radiation.cpp + eamxx_rrtmgp_process_interface.cpp shr_orb_mod_c2f.F90 ) diff --git a/components/eamxx/src/physics/rrtmgp/atmosphere_radiation.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp similarity index 72% rename from components/eamxx/src/physics/rrtmgp/atmosphere_radiation.cpp rename to components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 51dd0fbdaecf..0d7a933ec3e0 100644 --- a/components/eamxx/src/physics/rrtmgp/atmosphere_radiation.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -1,14 +1,18 @@ #include "physics/rrtmgp/scream_rrtmgp_interface.hpp" -#include "physics/rrtmgp/atmosphere_radiation.hpp" +#include "physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp" #include "physics/rrtmgp/rrtmgp_utils.hpp" #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" #include "physics/share/scream_trcmix.hpp" + +#include "share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp" #include "share/property_checks/field_within_interval_check.hpp" #include "share/util/scream_common_physics_functions.hpp" #include "share/util/scream_column_ops.hpp" + +#include "ekat/ekat_assert.hpp" + #include "cpp/rrtmgp/mo_gas_concentrations.h" #include "YAKL.h" -#include "ekat/ekat_assert.hpp" namespace scream { @@ -20,23 +24,24 @@ RRTMGPRadiation:: RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { - // Nothing to do here -} // RRTMGPRadiation::RRTMGPRadiation - -// ========================================================================================= -void RRTMGPRadiation::set_grids(const std::shared_ptr grids_manager) { - - using namespace ekat::units; - // Gather the active gases from the rrtmgp parameter list and assign to the m_gas_names vector. - auto active_gases = m_params.get>("active_gases"); + const auto& active_gases = m_params.get>("active_gases"); for (auto& it : active_gases) { // Make sure only unique names are added - if (std::find(m_gas_names.begin(), m_gas_names.end(), it) == m_gas_names.end()) { + if (not ekat::contains(m_gas_names,it)) { m_gas_names.push_back(it); + if (it=="o3") { + TraceGasesWorkaround::singleton().add_active_gas(it + "_volume_mix_ratio"); + } } } + m_ngas = m_gas_names.size(); +} + +void RRTMGPRadiation::set_grids(const std::shared_ptr grids_manager) { + + using namespace ekat::units; // Declare the set of fields used by rrtmgp auto kgkg = kg/kg; @@ -81,60 +86,88 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ FieldLayout scalar3d_swgpts_layout { {COL,SWGPT,LEV}, {m_ncol, m_nswgpts, m_nlay} }; FieldLayout scalar3d_lwgpts_layout { {COL,LWGPT,LEV}, {m_ncol, m_nlwgpts, m_nlay} }; - constexpr int ps = SCREAM_SMALL_PACK_SIZE; // Set required (input) fields here - add_field("p_mid" , scalar3d_layout_mid, Pa, grid_name, ps); - add_field("p_int", scalar3d_layout_int, Pa, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); + add_field("p_mid" , scalar3d_layout_mid, Pa, grid_name); + add_field("p_int", scalar3d_layout_int, Pa, grid_name); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); add_field("sfc_alb_dir_vis", scalar2d_layout, nondim, grid_name); add_field("sfc_alb_dir_nir", scalar2d_layout, nondim, grid_name); add_field("sfc_alb_dif_vis", scalar2d_layout, nondim, grid_name); add_field("sfc_alb_dif_nir", scalar2d_layout, nondim, grid_name); - add_field("qc", scalar3d_layout_mid, kgkg, grid_name, ps); - add_field("qi", scalar3d_layout_mid, kgkg, grid_name, ps); - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name, ps); - add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name, ps); - add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name, ps); - add_field("qv",scalar3d_layout_mid,kgkg,grid_name, ps); + add_field("qc", scalar3d_layout_mid, kgkg, grid_name); + add_field("nc", scalar3d_layout_mid, 1/kg, grid_name); + add_field("qi", scalar3d_layout_mid, kgkg, grid_name); + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name); + add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name); + add_field("qv",scalar3d_layout_mid,kgkg,grid_name); add_field("surf_lw_flux_up",scalar2d_layout,W/(m*m),grid_name); // Set of required gas concentration fields for (auto& it : m_gas_names) { // Add gas VOLUME mixing ratios (moles of gas / moles of air; what actually gets input to RRTMGP) if (it == "o3") { // o3 is read from file, or computed by chemistry - add_field(it + "_volume_mix_ratio", scalar3d_layout_mid, molmol, grid_name, ps); + add_field(it + "_volume_mix_ratio", scalar3d_layout_mid, molmol, grid_name); } else { - // the rest are computed from prescribed surface values - add_field(it + "_volume_mix_ratio", scalar3d_layout_mid, molmol, grid_name, ps); + // the rest are computed by RRTMGP from prescribed surface values + // NOTE: this may change at some point + add_field(it + "_volume_mix_ratio", scalar3d_layout_mid, molmol, grid_name); } } // Required aerosol optical properties from SPA m_do_aerosol_rad = m_params.get("do_aerosol_rad",true); if (m_do_aerosol_rad) { - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name, ps); - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name, ps); - add_field("aero_g_sw" , scalar3d_swband_layout, nondim, grid_name, ps); - add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name, ps); + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); + add_field("aero_g_sw" , scalar3d_swband_layout, nondim, grid_name); + add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); } + // Whether we do extra clean/clear sky calculations + m_extra_clnclrsky_diag = m_params.get("extra_clnclrsky_diag", false); + m_extra_clnsky_diag = m_params.get("extra_clnsky_diag", false); + // Set computed (output) fields - add_field("T_mid" , scalar3d_layout_mid, K , grid_name, ps); - add_field("SW_flux_dn", scalar3d_layout_int, Wm2, grid_name, "RESTART", ps); - add_field("SW_flux_up", scalar3d_layout_int, Wm2, grid_name, "RESTART", ps); - add_field("SW_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name, ps); - add_field("LW_flux_up", scalar3d_layout_int, Wm2, grid_name, "RESTART", ps); - add_field("LW_flux_dn", scalar3d_layout_int, Wm2, grid_name, "RESTART", ps); - add_field("SW_clrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name, ps); - add_field("SW_clrsky_flux_up", scalar3d_layout_int, Wm2, grid_name, ps); - add_field("SW_clrsky_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name, ps); - add_field("LW_clrsky_flux_up", scalar3d_layout_int, Wm2, grid_name, ps); - add_field("LW_clrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name, ps); - add_field("rad_heating_pdel", scalar3d_layout_mid, Pa*K/s, grid_name, "RESTART", ps); + add_field("T_mid" , scalar3d_layout_mid, K , grid_name); + add_field("SW_flux_dn", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_flux_up", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name); + add_field("LW_flux_up", scalar3d_layout_int, Wm2, grid_name); + add_field("LW_flux_dn", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clnclrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clnclrsky_flux_up", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clnclrsky_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clrsky_flux_up", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clrsky_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clnsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clnsky_flux_up", scalar3d_layout_int, Wm2, grid_name); + add_field("SW_clnsky_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name); + add_field("LW_clnclrsky_flux_up", scalar3d_layout_int, Wm2, grid_name); + add_field("LW_clnclrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); + add_field("LW_clrsky_flux_up", scalar3d_layout_int, Wm2, grid_name); + add_field("LW_clrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); + add_field("LW_clnsky_flux_up", scalar3d_layout_int, Wm2, grid_name); + add_field("LW_clnsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); + add_field("rad_heating_pdel", scalar3d_layout_mid, Pa*K/s, grid_name); // Cloud properties added as computed fields for diagnostic purposes - add_field("cldlow" , scalar2d_layout, nondim, grid_name, "RESTART"); - add_field("cldmed" , scalar2d_layout, nondim, grid_name, "RESTART"); - add_field("cldhgh" , scalar2d_layout, nondim, grid_name, "RESTART"); - add_field("cldtot" , scalar2d_layout, nondim, grid_name, "RESTART"); + add_field("cldlow" , scalar2d_layout, nondim, grid_name); + add_field("cldmed" , scalar2d_layout, nondim, grid_name); + add_field("cldhgh" , scalar2d_layout, nondim, grid_name); + add_field("cldtot" , scalar2d_layout, nondim, grid_name); + // 0.67 micron and 10.5 micron optical depth (needed for COSP) + add_field("dtau067" , scalar3d_layout_mid, nondim, grid_name); + add_field("dtau105" , scalar3d_layout_mid, nondim, grid_name); + add_field("sunlit" , scalar2d_layout , nondim, grid_name); + // Cloud-top diagnostics following AeroCOM recommendation + add_field("T_mid_at_cldtop", scalar2d_layout, K, grid_name); + add_field("p_mid_at_cldtop", scalar2d_layout, Pa, grid_name); + add_field("cldfrac_ice_at_cldtop", scalar2d_layout, nondim, grid_name); + add_field("cldfrac_liq_at_cldtop", scalar2d_layout, nondim, grid_name); + add_field("cldfrac_tot_at_cldtop", scalar2d_layout, nondim, grid_name); + add_field("cdnc_at_cldtop", scalar2d_layout, 1 / (m * m * m), grid_name); + add_field("eff_radius_qc_at_cldtop", scalar2d_layout, micron, grid_name); + add_field("eff_radius_qi_at_cldtop", scalar2d_layout, micron, grid_name); // Translation of variables from EAM // -------------------------------------------------------------- @@ -147,13 +180,12 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ // netsw sfc_flux_sw_net net (down - up) SW flux at surface // flwds sfc_flux_lw_dn downwelling LW flux at surface // -------------------------------------------------------------- - // These need to be added to restarts in the case of super-stepping - add_field("sfc_flux_dir_nir", scalar2d_layout, Wm2, grid_name, "RESTART"); - add_field("sfc_flux_dir_vis", scalar2d_layout, Wm2, grid_name, "RESTART"); - add_field("sfc_flux_dif_nir", scalar2d_layout, Wm2, grid_name, "RESTART"); - add_field("sfc_flux_dif_vis", scalar2d_layout, Wm2, grid_name, "RESTART"); - add_field("sfc_flux_sw_net" , scalar2d_layout, Wm2, grid_name, "RESTART"); - add_field("sfc_flux_lw_dn" , scalar2d_layout, Wm2, grid_name, "RESTART"); + add_field("sfc_flux_dir_nir", scalar2d_layout, Wm2, grid_name); + add_field("sfc_flux_dir_vis", scalar2d_layout, Wm2, grid_name); + add_field("sfc_flux_dif_nir", scalar2d_layout, Wm2, grid_name); + add_field("sfc_flux_dif_vis", scalar2d_layout, Wm2, grid_name); + add_field("sfc_flux_sw_net" , scalar2d_layout, Wm2, grid_name); + add_field("sfc_flux_lw_dn" , scalar2d_layout, Wm2, grid_name); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { @@ -215,10 +247,14 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.p_lay.totElems(); m_buffer.t_lay = decltype(m_buffer.t_lay)("t_lay", mem, m_col_chunk_size, m_nlay); mem += m_buffer.t_lay.totElems(); + m_buffer.z_del = decltype(m_buffer.z_del)("z_del", mem, m_col_chunk_size, m_nlay); + mem += m_buffer.z_del.totElems(); m_buffer.p_del = decltype(m_buffer.p_del)("p_del", mem, m_col_chunk_size, m_nlay); mem += m_buffer.p_del.totElems(); m_buffer.qc = decltype(m_buffer.qc)("qc", mem, m_col_chunk_size, m_nlay); mem += m_buffer.qc.totElems(); + m_buffer.nc = decltype(m_buffer.nc)("nc", mem, m_col_chunk_size, m_nlay); + mem += m_buffer.nc.totElems(); m_buffer.qi = decltype(m_buffer.qi)("qi", mem, m_col_chunk_size, m_nlay); mem += m_buffer.qi.totElems(); m_buffer.cldfrac_tot = decltype(m_buffer.cldfrac_tot)("cldfrac_tot", mem, m_col_chunk_size, m_nlay); @@ -237,11 +273,15 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.sw_heating.totElems(); m_buffer.lw_heating = decltype(m_buffer.lw_heating)("lw_heating", mem, m_col_chunk_size, m_nlay); mem += m_buffer.lw_heating.totElems(); - // 3d arrays m_buffer.p_lev = decltype(m_buffer.p_lev)("p_lev", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.p_lev.totElems(); m_buffer.t_lev = decltype(m_buffer.t_lev)("t_lev", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.t_lev.totElems(); + m_buffer.d_tint = decltype(m_buffer.d_tint)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.d_tint.size(); + m_buffer.d_dz = decltype(m_buffer.d_dz )(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.d_dz.size(); + // 3d arrays m_buffer.sw_flux_up = decltype(m_buffer.sw_flux_up)("sw_flux_up", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.sw_flux_up.totElems(); m_buffer.sw_flux_dn = decltype(m_buffer.sw_flux_dn)("sw_flux_dn", mem, m_col_chunk_size, m_nlay+1); @@ -252,16 +292,36 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.lw_flux_up.totElems(); m_buffer.lw_flux_dn = decltype(m_buffer.lw_flux_dn)("lw_flux_dn", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.lw_flux_dn.totElems(); + m_buffer.sw_clnclrsky_flux_up = decltype(m_buffer.sw_clnclrsky_flux_up)("sw_clnclrsky_flux_up", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnclrsky_flux_up.totElems(); + m_buffer.sw_clnclrsky_flux_dn = decltype(m_buffer.sw_clnclrsky_flux_dn)("sw_clnclrsky_flux_dn", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnclrsky_flux_dn.totElems(); + m_buffer.sw_clnclrsky_flux_dn_dir = decltype(m_buffer.sw_clnclrsky_flux_dn_dir)("sw_clnclrsky_flux_dn_dir", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnclrsky_flux_dn_dir.totElems(); m_buffer.sw_clrsky_flux_up = decltype(m_buffer.sw_clrsky_flux_up)("sw_clrsky_flux_up", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.sw_clrsky_flux_up.totElems(); m_buffer.sw_clrsky_flux_dn = decltype(m_buffer.sw_clrsky_flux_dn)("sw_clrsky_flux_dn", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.sw_clrsky_flux_dn.totElems(); m_buffer.sw_clrsky_flux_dn_dir = decltype(m_buffer.sw_clrsky_flux_dn_dir)("sw_clrsky_flux_dn_dir", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.sw_clrsky_flux_dn_dir.totElems(); + m_buffer.sw_clnsky_flux_up = decltype(m_buffer.sw_clnsky_flux_up)("sw_clnsky_flux_up", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnsky_flux_up.totElems(); + m_buffer.sw_clnsky_flux_dn = decltype(m_buffer.sw_clnsky_flux_dn)("sw_clnsky_flux_dn", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnsky_flux_dn.totElems(); + m_buffer.sw_clnsky_flux_dn_dir = decltype(m_buffer.sw_clnsky_flux_dn_dir)("sw_clnsky_flux_dn_dir", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnsky_flux_dn_dir.totElems(); + m_buffer.lw_clnclrsky_flux_up = decltype(m_buffer.lw_clnclrsky_flux_up)("lw_clnclrsky_flux_up", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clnclrsky_flux_up.totElems(); + m_buffer.lw_clnclrsky_flux_dn = decltype(m_buffer.lw_clnclrsky_flux_dn)("lw_clnclrsky_flux_dn", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clnclrsky_flux_dn.totElems(); m_buffer.lw_clrsky_flux_up = decltype(m_buffer.lw_clrsky_flux_up)("lw_clrsky_flux_up", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.lw_clrsky_flux_up.totElems(); m_buffer.lw_clrsky_flux_dn = decltype(m_buffer.lw_clrsky_flux_dn)("lw_clrsky_flux_dn", mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.lw_clrsky_flux_dn.totElems(); + m_buffer.lw_clnsky_flux_up = decltype(m_buffer.lw_clnsky_flux_up)("lw_clnsky_flux_up", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clnsky_flux_up.totElems(); + m_buffer.lw_clnsky_flux_dn = decltype(m_buffer.lw_clnsky_flux_dn)("lw_clnsky_flux_dn", mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clnsky_flux_dn.totElems(); // 3d arrays with nswbands dimension (shortwave fluxes by band) m_buffer.sw_bnd_flux_up = decltype(m_buffer.sw_bnd_flux_up)("sw_bnd_flux_up", mem, m_col_chunk_size, m_nlay+1, m_nswbands); mem += m_buffer.sw_bnd_flux_up.totElems(); @@ -295,6 +355,10 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.cld_tau_sw_gpt.totElems(); m_buffer.cld_tau_lw_gpt = decltype(m_buffer.cld_tau_lw_gpt)("cld_tau_lw_gpt", mem, m_col_chunk_size, m_nlay, m_nlwgpts); mem += m_buffer.cld_tau_lw_gpt.totElems(); + m_buffer.cld_tau_sw_bnd = decltype(m_buffer.cld_tau_sw_bnd)("cld_tau_sw_bnd", mem, m_col_chunk_size, m_nlay, m_nswbands); + mem += m_buffer.cld_tau_sw_bnd.totElems(); + m_buffer.cld_tau_lw_bnd = decltype(m_buffer.cld_tau_lw_bnd)("cld_tau_lw_bnd", mem, m_col_chunk_size, m_nlay, m_nlwbands); + mem += m_buffer.cld_tau_lw_bnd.totElems(); size_t used_mem = (reinterpret_cast(mem) - buffer_manager.get_memory())*sizeof(Real); EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for RRTMGPRadiation."); @@ -363,6 +427,16 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // Set property checks for fields in this process add_invariant_check(get_field_out("T_mid"),m_grid,100.0, 500.0,false); + + // VMR of n2 and co is currently prescribed as a constant value, read from file + if (has_computed_field("n2_volume_mix_ratio",m_grid->name())) { + auto n2_vmr = get_field_out("n2_volume_mix_ratio").get_view(); + Kokkos::deep_copy(n2_vmr, m_params.get("n2vmr", 0.7906)); + } + if (has_computed_field("co_volume_mix_ratio",m_grid->name())) { + auto co_vmr = get_field_out("co_volume_mix_ratio").get_view(); + Kokkos::deep_copy(co_vmr, m_params.get("covmr", 1.0e-7)); + } } // ========================================================================================= @@ -386,6 +460,7 @@ void RRTMGPRadiation::run_impl (const double dt) { auto d_sfc_alb_dif_nir = get_field_in("sfc_alb_dif_nir").get_view(); auto d_qv = get_field_in("qv").get_view(); auto d_qc = get_field_in("qc").get_view(); + auto d_nc = get_field_in("nc").get_view(); auto d_qi = get_field_in("qi").get_view(); auto d_cldfrac_tot = get_field_in("cldfrac_tot").get_view(); auto d_rel = get_field_in("eff_radius_qc").get_view(); @@ -393,34 +468,39 @@ void RRTMGPRadiation::run_impl (const double dt) { auto d_surf_lw_flux_up = get_field_in("surf_lw_flux_up").get_view(); // Output fields auto d_tmid = get_field_out("T_mid").get_view(); - using SmallPack = ekat::Pack; - const int n_lay_w_pack = SCREAM_SMALL_PACK_SIZE*ekat::npack(m_nlay); - view_3d_real d_aero_tau_sw("aero_tau_sw",m_ncol,m_nswbands,n_lay_w_pack); - view_3d_real d_aero_ssa_sw("aero_ssa_sw",m_ncol,m_nswbands,n_lay_w_pack); - view_3d_real d_aero_g_sw ("aero_g_sw" ,m_ncol,m_nswbands,n_lay_w_pack); - view_3d_real d_aero_tau_lw("aero_tau_lw",m_ncol,m_nlwbands,n_lay_w_pack); - if (m_do_aerosol_rad) { - Kokkos::deep_copy(d_aero_tau_sw,get_field_in("aero_tau_sw").get_view()); - Kokkos::deep_copy(d_aero_ssa_sw,get_field_in("aero_ssa_sw").get_view()); - Kokkos::deep_copy(d_aero_g_sw ,get_field_in("aero_g_sw" ).get_view()); - Kokkos::deep_copy(d_aero_tau_lw,get_field_in("aero_tau_lw").get_view()); - } else { - Kokkos::deep_copy(d_aero_tau_sw,0.0); - Kokkos::deep_copy(d_aero_ssa_sw,0.0); - Kokkos::deep_copy(d_aero_g_sw ,0.0); - Kokkos::deep_copy(d_aero_tau_lw,0.0); + // Aerosol optics only exist if m_do_aerosol_rad is true, so declare views and copy from FM if so + using view_3d = Field::view_dev_t; + view_3d d_aero_tau_sw; + view_3d d_aero_ssa_sw; + view_3d d_aero_g_sw; + view_3d d_aero_tau_lw; + if (m_do_aerosol_rad) { + d_aero_tau_sw = get_field_in("aero_tau_sw").get_view(); + d_aero_ssa_sw = get_field_in("aero_ssa_sw").get_view(); + d_aero_g_sw = get_field_in("aero_g_sw" ).get_view(); + d_aero_tau_lw = get_field_in("aero_tau_lw").get_view(); } auto d_sw_flux_up = get_field_out("SW_flux_up").get_view(); auto d_sw_flux_dn = get_field_out("SW_flux_dn").get_view(); auto d_sw_flux_dn_dir = get_field_out("SW_flux_dn_dir").get_view(); auto d_lw_flux_up = get_field_out("LW_flux_up").get_view(); auto d_lw_flux_dn = get_field_out("LW_flux_dn").get_view(); + auto d_sw_clnclrsky_flux_up = get_field_out("SW_clnclrsky_flux_up").get_view(); + auto d_sw_clnclrsky_flux_dn = get_field_out("SW_clnclrsky_flux_dn").get_view(); + auto d_sw_clnclrsky_flux_dn_dir = get_field_out("SW_clnclrsky_flux_dn_dir").get_view(); auto d_sw_clrsky_flux_up = get_field_out("SW_clrsky_flux_up").get_view(); auto d_sw_clrsky_flux_dn = get_field_out("SW_clrsky_flux_dn").get_view(); auto d_sw_clrsky_flux_dn_dir = get_field_out("SW_clrsky_flux_dn_dir").get_view(); + auto d_sw_clnsky_flux_up = get_field_out("SW_clnsky_flux_up").get_view(); + auto d_sw_clnsky_flux_dn = get_field_out("SW_clnsky_flux_dn").get_view(); + auto d_sw_clnsky_flux_dn_dir = get_field_out("SW_clnsky_flux_dn_dir").get_view(); + auto d_lw_clnclrsky_flux_up = get_field_out("LW_clnclrsky_flux_up").get_view(); + auto d_lw_clnclrsky_flux_dn = get_field_out("LW_clnclrsky_flux_dn").get_view(); auto d_lw_clrsky_flux_up = get_field_out("LW_clrsky_flux_up").get_view(); auto d_lw_clrsky_flux_dn = get_field_out("LW_clrsky_flux_dn").get_view(); + auto d_lw_clnsky_flux_up = get_field_out("LW_clnsky_flux_up").get_view(); + auto d_lw_clnsky_flux_dn = get_field_out("LW_clnsky_flux_dn").get_view(); auto d_rad_heating_pdel = get_field_out("rad_heating_pdel").get_view(); auto d_sfc_flux_dir_vis = get_field_out("sfc_flux_dir_vis").get_view(); auto d_sfc_flux_dir_nir = get_field_out("sfc_flux_dir_nir").get_view(); @@ -432,12 +512,34 @@ void RRTMGPRadiation::run_impl (const double dt) { auto d_cldmed = get_field_out("cldmed").get_view(); auto d_cldhgh = get_field_out("cldhgh").get_view(); auto d_cldtot = get_field_out("cldtot").get_view(); + // Outputs for COSP + auto d_dtau067 = get_field_out("dtau067").get_view(); + auto d_dtau105 = get_field_out("dtau105").get_view(); + auto d_sunlit = get_field_out("sunlit").get_view(); + + Kokkos::deep_copy(d_dtau067,0.0); + Kokkos::deep_copy(d_dtau105,0.0); + // Outputs for AeroCOM cloud-top diagnostics + auto d_T_mid_at_cldtop = get_field_out("T_mid_at_cldtop").get_view(); + auto d_p_mid_at_cldtop = get_field_out("p_mid_at_cldtop").get_view(); + auto d_cldfrac_ice_at_cldtop = + get_field_out("cldfrac_ice_at_cldtop").get_view(); + auto d_cldfrac_liq_at_cldtop = + get_field_out("cldfrac_liq_at_cldtop").get_view(); + auto d_cldfrac_tot_at_cldtop = + get_field_out("cldfrac_tot_at_cldtop").get_view(); + auto d_cdnc_at_cldtop = get_field_out("cdnc_at_cldtop").get_view(); + auto d_eff_radius_qc_at_cldtop = + get_field_out("eff_radius_qc_at_cldtop").get_view(); + auto d_eff_radius_qi_at_cldtop = + get_field_out("eff_radius_qi_at_cldtop").get_view(); constexpr auto stebol = PC::stebol; const auto nlay = m_nlay; const auto nlwbands = m_nlwbands; const auto nswbands = m_nswbands; const auto nlwgpts = m_nlwgpts; + const auto do_aerosol_rad = m_do_aerosol_rad; // Are we going to update fluxes and heating this step? auto ts = timestamp(); @@ -474,6 +576,50 @@ void RRTMGPRadiation::run_impl (const double dt) { shr_orb_decl_c2f(calday, eccen, mvelpp, lambm0, obliqr, &delta, &eccf); + // Precompute VMR for all gases, on all cols, before starting the chunks loop + // + // h2o is taken from qv + // o3 is computed elsewhere (either read from file or computed by chemistry); + // n2 and co are set to constants and are not handled by trcmix; + // the rest are handled by trcmix + const auto gas_mol_weights = m_gas_mol_weights; + for (int igas = 0; igas < m_ngas; igas++) { + auto name = m_gas_names[igas]; + + // We read o3 in as a vmr already. Also, n2 and co are currently set + // as a constant value, read from file during init. Skip these. + if (name=="o3" or name == "n2" or name == "co") continue; + + auto d_vmr = get_field_out(name + "_volume_mix_ratio").get_view(); + if (name == "h2o") { + // h2o is (wet) mass mixing ratio in FM, otherwise known as "qv", which we've already read in above + // Convert to vmr + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_ncol, m_nlay); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + d_vmr(icol,k) = PF::calculate_vmr_from_mmr(gas_mol_weights[igas],d_qv(icol,k),d_qv(icol,k)); + }); + }); + Kokkos::fence(); + } else { + // This gives (dry) mass mixing ratios + scream::physics::trcmix( + name, m_nlay, m_lat.get_view(), d_pmid, d_vmr, + m_co2vmr, m_n2ovmr, m_ch4vmr, m_f11vmr, m_f12vmr + ); + // Back out volume mixing ratios + const auto air_mol_weight = PC::MWdry; + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_ncol, m_nlay); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int i = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + d_vmr(i,k) = air_mol_weight / gas_mol_weights[igas] * d_vmr(i,k); + }); + }); + } + } + // Loop over each chunk of columns for (int ic=0; ic::get_default_team_policy(ncol, m_nlay); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); @@ -611,8 +770,10 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { p_lay(i+1,k+1) = d_pmid(icol,k); t_lay(i+1,k+1) = d_tmid(icol,k); + z_del(i+1,k+1) = d_dz(i,k); p_del(i+1,k+1) = d_pdel(icol,k); qc(i+1,k+1) = d_qc(icol,k); + nc(i+1,k+1) = d_nc(icol,k); qi(i+1,k+1) = d_qi(icol,k); rel(i+1,k+1) = d_rel(icol,k); rei(i+1,k+1) = d_rei(icol,k); @@ -624,18 +785,33 @@ void RRTMGPRadiation::run_impl (const double dt) { t_lev(i+1,nlay+1) = d_tint(i,nlay); // Note that RRTMGP expects ordering (col,lay,bnd) but the FM keeps things in (col,bnd,lay) order - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { - auto b = idx / nlay; - auto k = idx % nlay; - aero_tau_sw(i+1,k+1,b+1) = d_aero_tau_sw(icol,b,k); - aero_ssa_sw(i+1,k+1,b+1) = d_aero_ssa_sw(icol,b,k); - aero_g_sw (i+1,k+1,b+1) = d_aero_g_sw (icol,b,k); - }); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlwbands*nlay), [&] (const int&idx) { - auto b = idx / nlay; - auto k = idx % nlay; - aero_tau_lw(i+1,k+1,b+1) = d_aero_tau_lw(icol,b,k); - }); + if (do_aerosol_rad) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_sw(i+1,k+1,b+1) = d_aero_tau_sw(icol,b,k); + aero_ssa_sw(i+1,k+1,b+1) = d_aero_ssa_sw(icol,b,k); + aero_g_sw (i+1,k+1,b+1) = d_aero_g_sw (icol,b,k); + }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlwbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_lw(i+1,k+1,b+1) = d_aero_tau_lw(icol,b,k); + }); + } else { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_sw(i+1,k+1,b+1) = 0; + aero_ssa_sw(i+1,k+1,b+1) = 0; + aero_g_sw (i+1,k+1,b+1) = 0; + }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlwbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_lw(i+1,k+1,b+1) = 0; + }); + } }); } Kokkos::fence(); @@ -644,52 +820,14 @@ void RRTMGPRadiation::run_impl (const double dt) { // set_vmr requires the input array size to have the correct size, // and the last chunk may have less columns, so create a temp of // correct size that uses m_buffer.tmp2d's pointer - // - // h2o is taken from qv and requies no initialization here; - // o3 is computed elsewhere (either read from file or computed by chemistry); - // n2 and co are set to constants and are not handled by trcmix; - // the rest are handled by trcmix real2d tmp2d = subview_2d(m_buffer.tmp2d); - const auto gas_mol_weights = m_gas_mol_weights; for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; - auto d_vmr = get_field_out(name + "_volume_mix_ratio").get_view(); - if (name == "h2o") { - // h2o is (wet) mass mixing ratio in FM, otherwise known as "qv", which we've already read in above - // Convert to vmr - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - const int i = team.league_rank(); - const int icol = i + beg; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { - d_vmr(icol,k) = PF::calculate_vmr_from_mmr(gas_mol_weights[igas],d_qv(icol,k),d_qv(icol,k)); - }); - }); - Kokkos::fence(); - } else if (name == "o3") { - // We read o3 in as a vmr already - } else if (name == "n2") { - // n2 prescribed as a constant value - Kokkos::deep_copy(d_vmr, m_params.get("n2vmr", 0.7906)); - } else if (name == "co") { - // co prescribed as a constant value - Kokkos::deep_copy(d_vmr, m_params.get("covmr", 1.0e-7)); - } else { - // This gives (dry) mass mixing ratios - scream::physics::trcmix( - name, m_lat.get_view(), d_pmid, d_vmr, - m_co2vmr, m_n2ovmr, m_ch4vmr, m_f11vmr, m_f12vmr - ); - // Back out volume mixing ratios - const auto air_mol_weight = PC::MWdry; - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_ncol, m_nlay); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - const int i = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { - d_vmr(i,k) = air_mol_weight / gas_mol_weights[igas] * d_vmr(i,k); - }); - }); - } + auto full_name = name + "_volume_mix_ratio"; + + // 'o3' is marked as 'Required' rather than 'Computed', so we need to get the proper field + auto f = name=="o3" ? get_field_in(full_name) : get_field_out(full_name); + auto d_vmr = f.get_view(); // Copy to YAKL const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); @@ -778,11 +916,18 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dir, sfc_alb_dif, mu0, lwp, iwp, rel, rei, cldfrac_tot, aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw, + cld_tau_sw_bnd, cld_tau_lw_bnd, cld_tau_sw_gpt, cld_tau_lw_gpt, sw_flux_up , sw_flux_dn , sw_flux_dn_dir , lw_flux_up , lw_flux_dn, - sw_clrsky_flux_up, sw_clrsky_flux_dn, sw_clrsky_flux_dn_dir, lw_clrsky_flux_up, lw_clrsky_flux_dn, + sw_clnclrsky_flux_up, sw_clnclrsky_flux_dn, sw_clnclrsky_flux_dn_dir, + sw_clrsky_flux_up, sw_clrsky_flux_dn, sw_clrsky_flux_dn_dir, + sw_clnsky_flux_up, sw_clnsky_flux_dn, sw_clnsky_flux_dn_dir, + lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, + lw_clrsky_flux_up, lw_clrsky_flux_dn, + lw_clnsky_flux_up, lw_clnsky_flux_dn, sw_bnd_flux_up , sw_bnd_flux_dn , sw_bnd_flux_dir , lw_bnd_flux_up , lw_bnd_flux_dn, - eccf, m_atm_logger + eccf, m_atm_logger, + m_extra_clnclrsky_diag, m_extra_clnsky_diag ); // Update heating tendency @@ -829,10 +974,10 @@ void RRTMGPRadiation::run_impl (const double dt) { ); // Compute diagnostic total cloud area (vertically-projected cloud cover) - auto cldlow = real1d("cldlow", ncol); - auto cldmed = real1d("cldmed", ncol); - auto cldhgh = real1d("cldhgh", ncol); - auto cldtot = real1d("cldtot", ncol); + real1d cldlow ("cldlow", d_cldlow.data() + m_col_chunk_beg[ic], ncol); + real1d cldmed ("cldmed", d_cldmed.data() + m_col_chunk_beg[ic], ncol); + real1d cldhgh ("cldhgh", d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + real1d cldtot ("cldtot", d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably @@ -844,6 +989,27 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay, cld_tau_lw_gpt, cldhgh); rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); + // Get visible 0.67 micron band for COSP + auto idx_067 = rrtmgp::get_wavelength_index_sw(0.67e-6); + // Get IR 10.5 micron band for COSP + auto idx_105 = rrtmgp::get_wavelength_index_lw(10.5e-6); + + // Compute cloud-top diagnostics following AeroCOM recommendation + real1d T_mid_at_cldtop ("T_mid_at_cldtop", d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1d p_mid_at_cldtop ("p_mid_at_cldtop", d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1d cldfrac_ice_at_cldtop ("cldfrac_ice_at_cldtop", d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1d cldfrac_liq_at_cldtop ("cldfrac_liq_at_cldtop", d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1d cldfrac_tot_at_cldtop ("cldfrac_tot_at_cldtop", d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1d cdnc_at_cldtop ("cdnc_at_cldtop", d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1d eff_radius_qc_at_cldtop ("eff_radius_qc_at_cldtop", d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1d eff_radius_qi_at_cldtop ("eff_radius_qi_at_cldtop", d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + + rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, t_lay, p_lay, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, + nc, T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + // Copy output data back to FieldManager const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -855,22 +1021,38 @@ void RRTMGPRadiation::run_impl (const double dt) { d_sfc_flux_dif_vis(icol) = sfc_flux_dif_vis(i+1); d_sfc_flux_sw_net(icol) = sw_flux_dn(i+1,kbot) - sw_flux_up(i+1,kbot); d_sfc_flux_lw_dn(icol) = lw_flux_dn(i+1,kbot); - d_cldlow(icol) = cldlow(i+1); - d_cldmed(icol) = cldmed(i+1); - d_cldhgh(icol) = cldhgh(i+1); - d_cldtot(icol) = cldtot(i+1); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { d_sw_flux_up(icol,k) = sw_flux_up(i+1,k+1); d_sw_flux_dn(icol,k) = sw_flux_dn(i+1,k+1); d_sw_flux_dn_dir(icol,k) = sw_flux_dn_dir(i+1,k+1); d_lw_flux_up(icol,k) = lw_flux_up(i+1,k+1); d_lw_flux_dn(icol,k) = lw_flux_dn(i+1,k+1); + d_sw_clnclrsky_flux_up(icol,k) = sw_clnclrsky_flux_up(i+1,k+1); + d_sw_clnclrsky_flux_dn(icol,k) = sw_clnclrsky_flux_dn(i+1,k+1); + d_sw_clnclrsky_flux_dn_dir(icol,k) = sw_clnclrsky_flux_dn_dir(i+1,k+1); d_sw_clrsky_flux_up(icol,k) = sw_clrsky_flux_up(i+1,k+1); d_sw_clrsky_flux_dn(icol,k) = sw_clrsky_flux_dn(i+1,k+1); d_sw_clrsky_flux_dn_dir(icol,k) = sw_clrsky_flux_dn_dir(i+1,k+1); + d_sw_clnsky_flux_up(icol,k) = sw_clnsky_flux_up(i+1,k+1); + d_sw_clnsky_flux_dn(icol,k) = sw_clnsky_flux_dn(i+1,k+1); + d_sw_clnsky_flux_dn_dir(icol,k) = sw_clnsky_flux_dn_dir(i+1,k+1); + d_lw_clnclrsky_flux_up(icol,k) = lw_clnclrsky_flux_up(i+1,k+1); + d_lw_clnclrsky_flux_dn(icol,k) = lw_clnclrsky_flux_dn(i+1,k+1); d_lw_clrsky_flux_up(icol,k) = lw_clrsky_flux_up(i+1,k+1); d_lw_clrsky_flux_dn(icol,k) = lw_clrsky_flux_dn(i+1,k+1); + d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up(i+1,k+1); + d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn(i+1,k+1); }); + // Extract optical properties for COSP + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + d_dtau067(icol,k) = cld_tau_sw_bnd(i+1,k+1,idx_067); + d_dtau105(icol,k) = cld_tau_lw_bnd(i+1,k+1,idx_105); + }); + if (d_sw_clrsky_flux_dn(icol,0) > 0) { + d_sunlit(icol) = 1.0; + } else { + d_sunlit(icol) = 0.0; + } }); } // loop over chunk diff --git a/components/eamxx/src/physics/rrtmgp/atmosphere_radiation.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp similarity index 85% rename from components/eamxx/src/physics/rrtmgp/atmosphere_radiation.hpp rename to components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index e96908a523bd..f1321fd08289 100644 --- a/components/eamxx/src/physics/rrtmgp/atmosphere_radiation.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -24,7 +24,9 @@ class RRTMGPRadiation : public AtmosphereProcess { using KT = ekat::KokkosTypes; template - using uview_1d = Unmanaged>; + using uview_1d = Unmanaged>; + template + using uview_2d = Unmanaged>; // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -56,6 +58,9 @@ class RRTMGPRadiation : public AtmosphereProcess { // Whether we use aerosol forcing in radiation bool m_do_aerosol_rad; + // Whether we do extra aerosol forcing calls + bool m_extra_clnsky_diag; + bool m_extra_clnclrsky_diag; // The orbital year, used for zenith angle calculations: // If > 0, use constant orbital year for duration of simulation @@ -103,13 +108,13 @@ class RRTMGPRadiation : public AtmosphereProcess { // Structure for storing local variables initialized using the ATMBufferManager struct Buffer { static constexpr int num_1d_ncol = 10; - static constexpr int num_2d_nlay = 13; - static constexpr int num_2d_nlay_p1 = 12; + static constexpr int num_2d_nlay = 16; + static constexpr int num_2d_nlay_p1 = 23; static constexpr int num_2d_nswbands = 2; static constexpr int num_3d_nlev_nswbands = 4; static constexpr int num_3d_nlev_nlwbands = 2; - static constexpr int num_3d_nlay_nswbands = 3; - static constexpr int num_3d_nlay_nlwbands = 1; + static constexpr int num_3d_nlay_nswbands = 4; + static constexpr int num_3d_nlay_nlwbands = 2; static constexpr int num_3d_nlay_nswgpts = 1; static constexpr int num_3d_nlay_nlwgpts = 1; @@ -128,8 +133,10 @@ class RRTMGPRadiation : public AtmosphereProcess { // 2d size (ncol, nlay) real2d p_lay; real2d t_lay; + real2d z_del; real2d p_del; real2d qc; + real2d nc; real2d qi; real2d cldfrac_tot; real2d eff_radius_qc; @@ -139,6 +146,7 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d iwp; real2d sw_heating; real2d lw_heating; + uview_2d d_dz; // 2d size (ncol, nlay+1) real2d p_lev; @@ -148,11 +156,22 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sw_flux_dn_dir; real2d lw_flux_up; real2d lw_flux_dn; + real2d sw_clnclrsky_flux_up; + real2d sw_clnclrsky_flux_dn; + real2d sw_clnclrsky_flux_dn_dir; real2d sw_clrsky_flux_up; real2d sw_clrsky_flux_dn; real2d sw_clrsky_flux_dn_dir; + real2d sw_clnsky_flux_up; + real2d sw_clnsky_flux_dn; + real2d sw_clnsky_flux_dn_dir; + real2d lw_clnclrsky_flux_up; + real2d lw_clnclrsky_flux_dn; real2d lw_clrsky_flux_up; real2d lw_clrsky_flux_dn; + real2d lw_clnsky_flux_up; + real2d lw_clnsky_flux_dn; + uview_2d d_tint; // 3d size (ncol, nlay+1, nswbands) real3d sw_bnd_flux_up; @@ -174,9 +193,14 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_g_sw; real3d aero_tau_lw; + // 3d size (ncol, nlay, n[sw,lw]bnds) + real3d cld_tau_sw_bnd; + real3d cld_tau_lw_bnd; + // 3d size (ncol, nlay, n[sw,lw]gpts) real3d cld_tau_sw_gpt; real3d cld_tau_lw_gpt; + }; protected: diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index ce0f122081ae..ec3460293951 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -7,6 +7,8 @@ #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" #include "cpp/rte/mo_rte_sw.h" #include "cpp/rte/mo_rte_lw.h" +#include "physics/share/physics_constants.hpp" +#include "ekat/util/ekat_math_utils.hpp" namespace scream { void yakl_init () @@ -215,16 +217,22 @@ namespace scream { real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cldfrac, real3d &aer_tau_sw, real3d &aer_ssa_sw, real3d &aer_asm_sw, real3d &aer_tau_lw, + real3d &cld_tau_sw_bnd, real3d &cld_tau_lw_bnd, real3d &cld_tau_sw_gpt, real3d &cld_tau_lw_gpt, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dn_dir, real2d &lw_flux_up, real2d &lw_flux_dn, + real2d &sw_clnclrsky_flux_up, real2d &sw_clnclrsky_flux_dn, real2d &sw_clnclrsky_flux_dn_dir, real2d &sw_clrsky_flux_up, real2d &sw_clrsky_flux_dn, real2d &sw_clrsky_flux_dn_dir, + real2d &sw_clnsky_flux_up, real2d &sw_clnsky_flux_dn, real2d &sw_clnsky_flux_dn_dir, + real2d &lw_clnclrsky_flux_up, real2d &lw_clnclrsky_flux_dn, real2d &lw_clrsky_flux_up, real2d &lw_clrsky_flux_dn, + real2d &lw_clnsky_flux_up, real2d &lw_clnsky_flux_dn, real3d &sw_bnd_flux_up, real3d &sw_bnd_flux_dn, real3d &sw_bnd_flux_dn_dir, real3d &lw_bnd_flux_up, real3d &lw_bnd_flux_dn, const Real tsi_scaling, - const std::shared_ptr& logger) { + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { #ifdef SCREAM_RRTMGP_DEBUG // Sanity check inputs, and possibly repair @@ -249,11 +257,21 @@ namespace scream { fluxes_sw.bnd_flux_up = sw_bnd_flux_up; fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; + // Clean-clear-sky + FluxesBroadband clnclrsky_fluxes_sw; + clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; + clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; + clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; // Clear-sky FluxesBroadband clrsky_fluxes_sw; clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; + // Clean-sky + FluxesBroadband clnsky_fluxes_sw; + clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; + clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; + clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; // Setup pointers to RRTMGP LW fluxes FluxesByband fluxes_lw; @@ -261,10 +279,18 @@ namespace scream { fluxes_lw.flux_dn = lw_flux_dn; fluxes_lw.bnd_flux_up = lw_bnd_flux_up; fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; + // Clean-clear-sky + FluxesBroadband clnclrsky_fluxes_lw; + clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; + clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; // Clear-sky FluxesBroadband clrsky_fluxes_lw; clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; + // Clean-sky + FluxesBroadband clnsky_fluxes_lw; + clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; + clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; auto nswbands = k_dist_sw.get_nband(); auto nlwbands = k_dist_lw.get_nband(); @@ -295,10 +321,12 @@ namespace scream { check_range(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); #endif - // Convert cloud physical properties to optical properties for input to RRTMGP OpticalProps2str clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw, k_dist_sw, lwp, iwp, rel, rei); OpticalProps1scl clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw, k_dist_lw, lwp, iwp, rel, rei); + clouds_sw.tau.deep_copy_to(cld_tau_sw_bnd); + clouds_lw.tau.deep_copy_to(cld_tau_lw_bnd); + // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; // This implements the Monte Carlo Independing Column Approximation by mapping only a single // subcolumn (cloud state) to each gpoint. @@ -335,8 +363,9 @@ namespace scream { ncol, nlay, k_dist_sw, p_lay, t_lay, p_lev, t_lev, gas_concs, sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, - fluxes_sw, clrsky_fluxes_sw, - tsi_scaling, logger + fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, + tsi_scaling, logger, + extra_clnclrsky_diag, extra_clnsky_diag ); // Do longwave @@ -344,7 +373,8 @@ namespace scream { ncol, nlay, k_dist_lw, p_lay, t_lay, p_lev, t_lev, gas_concs, aerosol_lw, clouds_lw_gpt, - fluxes_lw, clrsky_fluxes_lw + fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, + extra_clnclrsky_diag, extra_clnsky_diag ); } @@ -573,9 +603,10 @@ namespace scream { GasConcs &gas_concs, real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, OpticalProps2str &aerosol, OpticalProps2str &clouds, - FluxesByband &fluxes, FluxesBroadband &clrsky_fluxes, + FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, const Real tsi_scaling, - const std::shared_ptr& logger) { + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Get problem sizes int nbnd = k_dist.get_nband(); @@ -589,18 +620,30 @@ namespace scream { auto &bnd_flux_up = fluxes.bnd_flux_up; auto &bnd_flux_dn = fluxes.bnd_flux_dn; auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; auto &clrsky_flux_up = clrsky_fluxes.flux_up; auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; // Reset fluxes to zero parallel_for(SimpleBounds<2>(nlay+1,ncol), YAKL_LAMBDA(int ilev, int icol) { flux_up (icol,ilev) = 0; flux_dn (icol,ilev) = 0; flux_dn_dir(icol,ilev) = 0; + clnclrsky_flux_up (icol,ilev) = 0; + clnclrsky_flux_dn (icol,ilev) = 0; + clnclrsky_flux_dn_dir(icol,ilev) = 0; clrsky_flux_up (icol,ilev) = 0; clrsky_flux_dn (icol,ilev) = 0; clrsky_flux_dn_dir(icol,ilev) = 0; + clnsky_flux_up (icol,ilev) = 0; + clnsky_flux_dn (icol,ilev) = 0; + clnsky_flux_dn_dir(icol,ilev) = 0; }); parallel_for(SimpleBounds<3>(nbnd,nlay+1,ncol), YAKL_LAMBDA(int ibnd, int ilev, int icol) { bnd_flux_up (icol,ilev,ibnd) = 0; @@ -713,6 +756,12 @@ namespace scream { OpticalProps2str optics; optics.alloc_2str(nday, nlay, k_dist); + OpticalProps2str optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_2str(nday, nlay, k_dist); + } + // Limit temperatures for gas optics look-up tables auto t_lay_limited = real2d("t_lay_limited", nday, nlay); limit_to_bounds(t_lay_day, k_dist_sw.get_temp_min(), k_dist_sw.get_temp_max(), t_lay_limited); @@ -723,6 +772,10 @@ namespace scream { bool top_at_1 = p_lay_host(1, 1) < p_lay_host(1, nlay); k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, optics, toa_flux); + if (extra_clnsky_diag) { + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, optics_no_aerosols, toa_flux); + } + #ifdef SCREAM_RRTMGP_DEBUG // Check gas optics @@ -736,6 +789,18 @@ namespace scream { toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); }); + if (extra_clnclrsky_diag) { + // Compute clear-clean-sky (just gas) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { + int icol = dayIndices(iday); + clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + // Combine gas and aerosol optics aerosol_day.delta_scale(); aerosol_day.increment(optics); @@ -770,6 +835,21 @@ namespace scream { bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); }); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds_day.increment(optics_no_aerosols); + // Compute cleansky (gas + clouds) fluxes on daytime columns + rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { + int icol = dayIndices(iday); + clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + } void rrtmgp_lw( @@ -779,14 +859,51 @@ namespace scream { GasConcs &gas_concs, OpticalProps1scl &aerosol, OpticalProps1scl &clouds, - FluxesByband &fluxes, FluxesBroadband &clrsky_fluxes) { + FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Problem size int nbnd = k_dist.get_nband(); + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + + // Reset fluxes to zero + parallel_for( + SimpleBounds<2>(nlay + 1, ncol), YAKL_LAMBDA(int ilev, int icol) { + flux_up(icol, ilev) = 0; + flux_dn(icol, ilev) = 0; + clnclrsky_flux_up(icol, ilev) = 0; + clnclrsky_flux_dn(icol, ilev) = 0; + clrsky_flux_up(icol, ilev) = 0; + clrsky_flux_dn(icol, ilev) = 0; + clnsky_flux_up(icol, ilev) = 0; + clnsky_flux_dn(icol, ilev) = 0; + }); + parallel_for( + SimpleBounds<3>(nbnd, nlay + 1, ncol), + YAKL_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up(icol, ilev, ibnd) = 0; + bnd_flux_dn(icol, ilev, ibnd) = 0; + }); + // Allocate space for optical properties OpticalProps1scl optics; optics.alloc_1scl(ncol, nlay, k_dist); + OpticalProps1scl optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); + } // Boundary conditions SourceFuncLW lw_sources; @@ -833,12 +950,20 @@ namespace scream { // Do gas optics k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, optics, lw_sources, real2d(), t_lev_limited); + if (extra_clnsky_diag) { + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, optics_no_aerosols, lw_sources, real2d(), t_lev_limited); + } #ifdef SCREAM_RRTMGP_DEBUG // Check gas optics check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); #endif + if (extra_clnclrsky_diag) { + // Compute clean-clear-sky fluxes before we add in aerosols and clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); + } + // Combine gas and aerosol optics aerosol.increment(optics); @@ -851,6 +976,13 @@ namespace scream { // Compute allsky fluxes rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds.increment(optics_no_aerosols); + // Compute clean-sky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); + } + } void compute_cloud_area( @@ -878,5 +1010,128 @@ namespace scream { }); } + int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } + + int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } + + int get_wavelength_index(OpticalProps &kdist, double wavelength) { + // Get wavelength bounds for all wavelength bands + auto wavelength_bounds = kdist.get_band_lims_wavelength(); + + // Find the band index for the specified wavelength + // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength + // in units of meters, we need a conversion factor of 10^2 + int nbnds = kdist.get_nband(); + yakl::ScalarLiveOut band_index(-1); + yakl::fortran::parallel_for(SimpleBounds<1>(nbnds), YAKL_LAMBDA(int ibnd) { + if (wavelength_bounds(1,ibnd) < wavelength_bounds(2,ibnd)) { + if (wavelength_bounds(1,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(2,ibnd)) { + band_index = ibnd; + } + } else { + if (wavelength_bounds(1,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(2,ibnd)) { + band_index = ibnd; + } + } + }); + return band_index.hostRead(); + } + + void compute_aerocom_cloudtop( + int ncol, int nlay, const real2d &tmid, const real2d &pmid, + const real2d &p_del, const real2d &z_del, const real2d &qc, + const real2d &qi, const real2d &rel, const real2d &rei, + const real2d &cldfrac_tot, const real2d &nc, + real1d &T_mid_at_cldtop, real1d &p_mid_at_cldtop, + real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, + real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, + real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop) { + /* The goal of this routine is to calculate properties at cloud top + * based on the AeroCOM recommendation. See reference for routine + * get_subcolumn_mask above, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. + */ + // Set outputs to zero + memset(T_mid_at_cldtop, 0.0); + memset(p_mid_at_cldtop, 0.0); + memset(cldfrac_ice_at_cldtop, 0.0); + memset(cldfrac_liq_at_cldtop, 0.0); + memset(cldfrac_tot_at_cldtop, 0.0); + memset(cdnc_at_cldtop, 0.0); + memset(eff_radius_qc_at_cldtop, 0.0); + memset(eff_radius_qi_at_cldtop, 0.0); + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto aerocom_clr = real1d("aerocom_clr", ncol); + memset(aerocom_clr, 1.0); + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + // TODO: move tunable constant to namelist + constexpr real q_threshold = 0.0; // BAD_CONSTANT! + // TODO: move tunable constant to namelist + constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! + // Loop over all columns in parallel + yakl::fortran::parallel_for( + SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds + for(int ilay = 2; ilay <= nlay; ++ilay) { + // Only do the calculation if certain conditions are met + if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && + (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + aerocom_clr(icol) * + (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), + cldfrac_tot(icol, ilay))) / + (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = + qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; + // p_mid_at_cldtop + p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + cldfrac_ice_at_cldtop(icol) += + (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / + z_del(icol, ilay) / physconst::gravit / + cldfrac_tot(icol, ilay); + cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtop + eff_radius_qc_at_cldtop(icol) += + rel(icol, ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtop + eff_radius_qi_at_cldtop(icol) += + rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + aerocom_clr(icol) = aerocom_tmp; + } + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); + }); + } } // namespace rrtmgp } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 99655b4eab80..65e165422865 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -68,15 +68,21 @@ namespace scream { real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cldfrac, real3d &aer_tau_sw, real3d &aer_ssa_sw, real3d &aer_asm_sw, real3d &aer_tau_lw, + real3d &cld_tau_sw_bnd, real3d &cld_tau_lw_bnd, real3d &cld_tau_sw_gpt, real3d &cld_tau_lw_gpt, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dn_dir, real2d &lw_flux_up, real2d &lw_flux_dn, + real2d &sw_clnclrsky_flux_up, real2d &sw_clnclrsky_flux_dn, real2d &sw_clnclrsky_flux_dn_dir, real2d &sw_clrsky_flux_up, real2d &sw_clrsky_flux_dn, real2d &sw_clrsky_flux_dn_dir, + real2d &sw_clnsky_flux_up, real2d &sw_clnsky_flux_dn, real2d &sw_clnsky_flux_dn_dir, + real2d &lw_clnclrsky_flux_up, real2d &lw_clnclrsky_flux_dn, real2d &lw_clrsky_flux_up, real2d &lw_clrsky_flux_dn, + real2d &lw_clnsky_flux_up, real2d &lw_clnsky_flux_dn, real3d &sw_bnd_flux_up, real3d &sw_bnd_flux_dn, real3d &sw_bnd_flux_dn_dir, real3d &lw_bnd_flux_up, real3d &lw_bnd_flux_dn, const Real tsi_scaling, - const std::shared_ptr& logger); + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); /* * Perform any clean-up tasks */ @@ -90,8 +96,10 @@ namespace scream { GasConcs &gas_concs, real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, OpticalProps2str &aerosol, OpticalProps2str &clouds, - FluxesByband &fluxes, FluxesBroadband &clrsky_fluxes, const Real tsi_scaling, - const std::shared_ptr& logger); + FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); /* * Longwave driver (called by rrtmgp_main) */ @@ -101,7 +109,8 @@ namespace scream { real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, GasConcs &gas_concs, OpticalProps1scl &aerosol, OpticalProps1scl &clouds, - FluxesByband &fluxes, FluxesBroadband &clrsky_fluxes); + FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); /* * Return a subcolumn mask consistent with a specified overlap assumption */ @@ -112,6 +121,18 @@ namespace scream { void compute_cloud_area( int ncol, int nlay, int ngpt, Real pmin, Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); + /* + * Return select cloud-top diagnostics following AeroCOM recommendation + */ + void compute_aerocom_cloudtop( + int ncol, int nlay, const real2d &tmid, const real2d &pmid, + const real2d &p_del, const real2d &z_del, const real2d &qc, + const real2d &qi, const real2d &rel, const real2d &rei, + const real2d &cldfrac_tot, const real2d &nc, + real1d &T_mid_at_cldtop, real1d &p_mid_at_cldtop, + real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, + real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, + real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop); /* * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area @@ -154,6 +175,9 @@ namespace scream { }); } + int get_wavelength_index(OpticalProps &kdist, double wavelength); + int get_wavelength_index_sw(double wavelength); + int get_wavelength_index_lw(double wavelength); } // namespace rrtmgp } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/simple_netcdf.hpp b/components/eamxx/src/physics/rrtmgp/simple_netcdf.hpp index 5bdfc55f16c1..fb965623bc9c 100644 --- a/components/eamxx/src/physics/rrtmgp/simple_netcdf.hpp +++ b/components/eamxx/src/physics/rrtmgp/simple_netcdf.hpp @@ -81,7 +81,7 @@ namespace simple_netcdf { std::vector dimSizes(ndims); size_t dimsize; for (int i = 0; i < ndims; i++) { - handle_error(nc_inq_dimlen(ncid, dimids[i], &dimsize), __FILE__, __LINE__); + handle_error(nc_inq_dimlen(ncid, dimids[i], &dimsize), __FILE__, __LINE__); dimSizes[i] = dimsize; } @@ -95,7 +95,7 @@ namespace simple_netcdf { // Read variable data if (myMem == memDevice) { - auto arrHost = arr.createHostCopy(); + auto arrHost = arr.createHostObject(); if (std::is_same::value) { // Create boolean array from integer arrays Array tmp("tmp",dimSizes); @@ -222,7 +222,7 @@ namespace simple_netcdf { handle_error(nc_put_var(ncid, varid, arr), __FILE__, __LINE__); } - template + template void write(Array const &arr, std::string varName, std::vector dimNames) { // Make sure length of dimension names is equal to rank of array diff --git a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt index 89014903520a..8a6a15948b4c 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt @@ -1,12 +1,9 @@ # NOTE: tests inside this if statement won't be built in a baselines-only build if (NOT SCREAM_BASELINES_ONLY) - # Required libraries - set (NEED_LIBS scream_rrtmgp rrtmgp_test_utils) - # Build baseline code add_executable(generate_baseline generate_baseline.cpp) - target_link_libraries(generate_baseline PUBLIC ${NEED_LIBS}) + target_link_libraries(generate_baseline PUBLIC scream_rrtmgp rrtmgp_test_utils) # Generate allsky baseline with the usual cmake custom command-target pair pattern # Note: these "baselines" are not to compare scream with a previous version, but @@ -21,15 +18,16 @@ if (NOT SCREAM_BASELINES_ONLY) DEPENDS ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc ) - CreateUnitTest( - rrtmgp_tests rrtmgp_tests.cpp "${NEED_LIBS}" LABELS "rrtmgp;physics" + CreateUnitTest(rrtmgp_tests rrtmgp_tests.cpp + LIBS scream_rrtmgp rrtmgp_test_utils + LABELS "rrtmgp;physics" EXE_ARGS "-i ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc -b ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc" EXCLUDE_MAIN_CPP ) add_dependencies (rrtmgp_tests rrtmgp_allsky_baseline.nc) - CreateUnitTest( - rrtmgp_unit_tests rrtmgp_unit_tests.cpp "${NEED_LIBS}" LABELS "rrtmgp;physics" + CreateUnitTest(rrtmgp_unit_tests rrtmgp_unit_tests.cpp + LIBS scream_rrtmgp rrtmgp_test_utils + LABELS "rrtmgp;physics" ) - endif() diff --git a/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp b/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp index 39f4e0efd3e6..4e678c7a825c 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp @@ -108,11 +108,21 @@ int main (int argc, char** argv) { real2d sw_flux_dn_dir("sw_flux_dn_dir", ncol, nlay+1); real2d lw_flux_up ("lw_flux_up" , ncol, nlay+1); real2d lw_flux_dn ("lw_flux_dn" , ncol, nlay+1); + real2d sw_clnclrsky_flux_up ("sw_clnclrsky_flux_up" , ncol, nlay+1); + real2d sw_clnclrsky_flux_dn ("sw_clnclrsky_flux_dn" , ncol, nlay+1); + real2d sw_clnclrsky_flux_dn_dir("sw_clnclrsky_flux_dn_dir", ncol, nlay+1); real2d sw_clrsky_flux_up ("sw_clrsky_flux_up" , ncol, nlay+1); real2d sw_clrsky_flux_dn ("sw_clrsky_flux_dn" , ncol, nlay+1); real2d sw_clrsky_flux_dn_dir("sw_clrsky_flux_dn_dir", ncol, nlay+1); + real2d sw_clnsky_flux_up ("sw_clnsky_flux_up" , ncol, nlay+1); + real2d sw_clnsky_flux_dn ("sw_clnsky_flux_dn" , ncol, nlay+1); + real2d sw_clnsky_flux_dn_dir("sw_clnsky_flux_dn_dir", ncol, nlay+1); + real2d lw_clnclrsky_flux_up ("lw_clnclrsky_flux_up" , ncol, nlay+1); + real2d lw_clnclrsky_flux_dn ("lw_clnclrsky_flux_dn" , ncol, nlay+1); real2d lw_clrsky_flux_up ("lw_clrsky_flux_up" , ncol, nlay+1); real2d lw_clrsky_flux_dn ("lw_clrsky_flux_dn" , ncol, nlay+1); + real2d lw_clnsky_flux_up ("lw_clnsky_flux_up" , ncol, nlay+1); + real2d lw_clnsky_flux_dn ("lw_clnsky_flux_dn" , ncol, nlay+1); real3d sw_bnd_flux_up ("sw_bnd_flux_up" , ncol, nlay+1, nswbands); real3d sw_bnd_flux_dn ("sw_bnd_flux_dn" , ncol, nlay+1, nswbands); real3d sw_bnd_flux_dir("sw_bnd_flux_dir", ncol, nlay+1, nswbands); @@ -146,6 +156,8 @@ int main (int argc, char** argv) { // TODO: provide as inputs consistent with how aerosol is treated? const auto nswgpts = scream::rrtmgp::k_dist_sw.get_ngpt(); const auto nlwgpts = scream::rrtmgp::k_dist_lw.get_ngpt(); + auto cld_tau_sw_bnd = real3d("cld_tau_sw_bnd", ncol, nlay, nswbands); + auto cld_tau_lw_bnd = real3d("cld_tau_lw_bnd", ncol, nlay, nlwbands); auto cld_tau_sw = real3d("cld_tau_sw", ncol, nlay, nswgpts); auto cld_tau_lw = real3d("cld_tau_lw", ncol, nlay, nlwgpts); @@ -160,11 +172,16 @@ int main (int argc, char** argv) { sfc_alb_dir, sfc_alb_dif, mu0, lwp, iwp, rel, rei, cld, aer_tau_sw, aer_ssa_sw, aer_asm_sw, aer_tau_lw, + cld_tau_sw_bnd, cld_tau_lw_bnd, cld_tau_sw, cld_tau_lw, // outputs sw_flux_up, sw_flux_dn, sw_flux_dn_dir, lw_flux_up, lw_flux_dn, + sw_clnclrsky_flux_up, sw_clnclrsky_flux_dn, sw_clnclrsky_flux_dn_dir, sw_clrsky_flux_up, sw_clrsky_flux_dn, sw_clrsky_flux_dn_dir, + sw_clnsky_flux_up, sw_clnsky_flux_dn, sw_clnsky_flux_dn_dir, + lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, + lw_clnsky_flux_up, lw_clnsky_flux_dn, sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn, tsi_scaling, logger @@ -204,6 +221,8 @@ int main (int argc, char** argv) { aer_tau_lw.deallocate(); cld_tau_sw.deallocate(); cld_tau_lw.deallocate(); + cld_tau_sw_bnd.deallocate(); + cld_tau_lw_bnd.deallocate(); sw_flux_up_ref.deallocate(); sw_flux_dn_ref.deallocate(); sw_flux_dn_dir_ref.deallocate(); @@ -214,11 +233,21 @@ int main (int argc, char** argv) { sw_flux_dn_dir.deallocate(); lw_flux_up.deallocate(); lw_flux_dn.deallocate(); + sw_clnclrsky_flux_up.deallocate(); + sw_clnclrsky_flux_dn.deallocate(); + sw_clnclrsky_flux_dn_dir.deallocate(); sw_clrsky_flux_up.deallocate(); sw_clrsky_flux_dn.deallocate(); sw_clrsky_flux_dn_dir.deallocate(); + sw_clnsky_flux_up.deallocate(); + sw_clnsky_flux_dn.deallocate(); + sw_clnsky_flux_dn_dir.deallocate(); + lw_clnclrsky_flux_up.deallocate(); + lw_clnclrsky_flux_dn.deallocate(); lw_clrsky_flux_up.deallocate(); lw_clrsky_flux_dn.deallocate(); + lw_clnsky_flux_up.deallocate(); + lw_clnsky_flux_dn.deallocate(); sw_bnd_flux_up.deallocate(); sw_bnd_flux_dn.deallocate(); sw_bnd_flux_dir.deallocate(); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index ca3d45673eb9..8b2ddbf373a7 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -142,11 +142,21 @@ int run(int argc, char** argv) { real2d sw_flux_dir("sw_flux_dir", ncol, nlay+1); real2d lw_flux_up ("lw_flux_up" , ncol, nlay+1); real2d lw_flux_dn ("lw_flux_dn" , ncol, nlay+1); + real2d sw_clnclrsky_flux_up ("sw_clnclrsky_flux_up" , ncol, nlay+1); + real2d sw_clnclrsky_flux_dn ("sw_clnclrsky_flux_dn" , ncol, nlay+1); + real2d sw_clnclrsky_flux_dir("sw_clnclrsky_flux_dir", ncol, nlay+1); real2d sw_clrsky_flux_up ("sw_clrsky_flux_up" , ncol, nlay+1); real2d sw_clrsky_flux_dn ("sw_clrsky_flux_dn" , ncol, nlay+1); real2d sw_clrsky_flux_dir("sw_clrsky_flux_dir", ncol, nlay+1); + real2d sw_clnsky_flux_up ("sw_clnsky_flux_up" , ncol, nlay+1); + real2d sw_clnsky_flux_dn ("sw_clnsky_flux_dn" , ncol, nlay+1); + real2d sw_clnsky_flux_dir("sw_clnsky_flux_dir", ncol, nlay+1); + real2d lw_clnclrsky_flux_up ("lw_clnclrsky_flux_up" , ncol, nlay+1); + real2d lw_clnclrsky_flux_dn ("lw_clnclrsky_flux_dn" , ncol, nlay+1); real2d lw_clrsky_flux_up ("lw_clrsky_flux_up" , ncol, nlay+1); real2d lw_clrsky_flux_dn ("lw_clrsky_flux_dn" , ncol, nlay+1); + real2d lw_clnsky_flux_up ("lw_clnsky_flux_up" , ncol, nlay+1); + real2d lw_clnsky_flux_dn ("lw_clnsky_flux_dn" , ncol, nlay+1); real3d sw_bnd_flux_up ("sw_bnd_flux_up" , ncol, nlay+1, nswbands); real3d sw_bnd_flux_dn ("sw_bnd_flux_dn" , ncol, nlay+1, nswbands); real3d sw_bnd_flux_dir("sw_bnd_flux_dir", ncol, nlay+1, nswbands); @@ -180,6 +190,8 @@ int run(int argc, char** argv) { // TODO: provide as inputs consistent with how aerosol is treated? const auto nswgpts = scream::rrtmgp::k_dist_sw.get_ngpt(); const auto nlwgpts = scream::rrtmgp::k_dist_lw.get_ngpt(); + auto cld_tau_sw_bnd = real3d("cld_tau_sw_bnd", ncol, nlay, nswbands); + auto cld_tau_lw_bnd = real3d("cld_tau_lw_bnd", ncol, nlay, nlwbands); auto cld_tau_sw = real3d("cld_tau_sw", ncol, nlay, nswgpts); auto cld_tau_lw = real3d("cld_tau_lw", ncol, nlay, nlwgpts); @@ -192,13 +204,21 @@ int run(int argc, char** argv) { sfc_alb_dir, sfc_alb_dif, mu0, lwp, iwp, rel, rei, cld, aer_tau_sw, aer_ssa_sw, aer_asm_sw, aer_tau_lw, + cld_tau_sw_bnd, cld_tau_lw_bnd, // outputs cld_tau_sw, cld_tau_lw, // outputs sw_flux_up, sw_flux_dn, sw_flux_dir, lw_flux_up, lw_flux_dn, + sw_clnclrsky_flux_up, sw_clnclrsky_flux_dn, sw_clnclrsky_flux_dir, sw_clrsky_flux_up, sw_clrsky_flux_dn, sw_clrsky_flux_dir, + sw_clnsky_flux_up, sw_clnsky_flux_dn, sw_clnsky_flux_dir, + lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, + lw_clnsky_flux_up, lw_clnsky_flux_dn, sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, - lw_bnd_flux_up, lw_bnd_flux_dn, tsi_scaling, logger); + lw_bnd_flux_up, lw_bnd_flux_dn, tsi_scaling, logger, + true, true // extra_clnclrsky_diag, extra_clnsky_diag + // set them both to true because we are testing them below + ); // Check values against baseline logger->info("Check values...\n"); @@ -214,6 +234,19 @@ int run(int argc, char** argv) { if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; + // Because the aerosol optical properties are all set to zero, these fluxes must be equal + if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + + logger->info("Cleaning up...\n"); // Clean up or else YAKL will throw errors scream::rrtmgp::rrtmgp_finalize(); @@ -227,11 +260,21 @@ int run(int argc, char** argv) { sw_flux_dir.deallocate(); lw_flux_up.deallocate(); lw_flux_dn.deallocate(); + sw_clnclrsky_flux_up.deallocate(); + sw_clnclrsky_flux_dn.deallocate(); + sw_clnclrsky_flux_dir.deallocate(); sw_clrsky_flux_up.deallocate(); sw_clrsky_flux_dn.deallocate(); sw_clrsky_flux_dir.deallocate(); + sw_clnsky_flux_up.deallocate(); + sw_clnsky_flux_dn.deallocate(); + sw_clnsky_flux_dir.deallocate(); + lw_clnclrsky_flux_up.deallocate(); + lw_clnclrsky_flux_dn.deallocate(); lw_clrsky_flux_up.deallocate(); lw_clrsky_flux_dn.deallocate(); + lw_clnsky_flux_up.deallocate(); + lw_clnsky_flux_dn.deallocate(); sw_bnd_flux_up.deallocate(); sw_bnd_flux_dn.deallocate(); sw_bnd_flux_dir.deallocate(); @@ -260,6 +303,8 @@ int run(int argc, char** argv) { aer_tau_lw.deallocate(); cld_tau_sw.deallocate(); cld_tau_lw.deallocate(); + cld_tau_sw_bnd.deallocate(); + cld_tau_lw_bnd.deallocate(); yakl::finalize(); return nerr != 0 ? 1 : 0; diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 5301a0cf6f5d..f737c2e249c0 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -4,6 +4,7 @@ #include "YAKL.h" #include "physics/share/physics_constants.hpp" #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" +#include "physics/rrtmgp/mo_load_coefficients.h" // Names of input files we will need. std::string coefficients_file_sw = SCREAM_DATA_DIR "/init/rrtmgp-data-sw-g112-210809.nc"; @@ -619,3 +620,225 @@ TEST_CASE("rrtmgp_cloud_area") { cldtot.deallocate(); yakl::finalize(); } + +TEST_CASE("rrtmgp_aerocom_cloudtop") { + // Initialize YAKL + if(!yakl::isInitialized()) { + yakl::init(); + } + // Create dummy data + const int ncol = 1; + const int nlay = 9; + // Set up input fields + auto tmid = real2d("tmid", ncol, nlay); + auto pmid = real2d("pmid", ncol, nlay); + auto p_del = real2d("p_del", ncol, nlay); + auto z_del = real2d("z_del", ncol, nlay); + auto qc = real2d("qc", ncol, nlay); + auto qi = real2d("qi", ncol, nlay); + auto rel = real2d("rel", ncol, nlay); + auto rei = real2d("rei", ncol, nlay); + auto cldfrac_tot = real2d("cldfrac_tot", ncol, nlay); + auto nc = real2d("nc", ncol, nlay); + // Set up output fields + auto tmid_at_cldtop = real1d("tmid_at_cldtop", ncol); + auto pmid_at_cldtop = real1d("pmid_at_cldtop", ncol); + auto cldfrac_ice_at_cldtop = real1d("cldfrac_ice_at_cldtop", ncol); + auto cldfrac_liq_at_cldtop = real1d("cldfrac_liq_at_cldtop", ncol); + auto cldfrac_tot_at_cldtop = real1d("cldfrac_tot_at_cldtop", ncol); + auto cdnc_at_cldtop = real1d("cdnc_at_cldtop", ncol); + auto eff_radius_qc_at_cldtop = real1d("eff_radius_qc_at_cldtop", ncol); + auto eff_radius_qi_at_cldtop = real1d("eff_radius_qi_at_cldtop", ncol); + + // Case 1: if no clouds, everything goes to zero + memset(tmid, 300.0); + memset(pmid, 100.0); + memset(p_del, 10.0); + memset(z_del, 100.0); + memset(qc, 1.0); + memset(qi, 1.0); + memset(cldfrac_tot, 0.0); + memset(nc, 5.0); + memset(rel, 10.0); + memset(rei, 10.0); + // Call the function + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + // Check the results + REQUIRE(tmid_at_cldtop.createHostCopy()(1) == 0.0); + REQUIRE(pmid_at_cldtop.createHostCopy()(1) == 0.0); + REQUIRE(cldfrac_tot_at_cldtop.createHostCopy()(1) == 0.0); + REQUIRE(cldfrac_liq_at_cldtop.createHostCopy()(1) == 0.0); + REQUIRE(cldfrac_ice_at_cldtop.createHostCopy()(1) == 0.0); + REQUIRE(cdnc_at_cldtop.createHostCopy()(1) == 0.0); + REQUIRE(eff_radius_qc_at_cldtop.createHostCopy()(1) == 0.0); + REQUIRE(eff_radius_qi_at_cldtop.createHostCopy()(1) == 0.0); + + // Case 2: if all clouds, everything goes to 1 * its value + memset(cldfrac_tot, 1.0); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(tmid_at_cldtop.createHostCopy()(1) == 300.0); + REQUIRE(pmid_at_cldtop.createHostCopy()(1) == 100.0); + REQUIRE(cldfrac_tot_at_cldtop.createHostCopy()(1) == 1.0); + REQUIRE(cldfrac_liq_at_cldtop.createHostCopy()(1) == 0.5); + REQUIRE(cldfrac_ice_at_cldtop.createHostCopy()(1) == 0.5); + REQUIRE(cdnc_at_cldtop.createHostCopy()(1) > 0.0); + REQUIRE(eff_radius_qc_at_cldtop.createHostCopy()(1) > 0.0); + REQUIRE(eff_radius_qi_at_cldtop.createHostCopy()(1) > 0.0); + + // Case 3: test max overlap (if contiguous cloudy layers, then max) + memset(cldfrac_tot, 0.0); + yakl::fortran::parallel_for( + 1, YAKL_LAMBDA(int /* dummy */) { + cldfrac_tot(1, 2) = 0.5; + cldfrac_tot(1, 3) = 0.7; + cldfrac_tot(1, 4) = 0.3; + cldfrac_tot(1, 5) = 0.2; + }); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(cldfrac_tot_at_cldtop.createHostCopy()(1) == .7); + + // Case 3xtra: test max overlap + // This case produces >0.7 due to slight enhancement in the presence of a + // local minimum (0.1 is the local minimum between 0.2 and 0.4) + yakl::fortran::parallel_for( + 1, YAKL_LAMBDA(int /* dummy */) { + cldfrac_tot(1, 5) = 0.1; + cldfrac_tot(1, 6) = 0.4; + cldfrac_tot(1, 7) = 0.2; + }); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(cldfrac_tot_at_cldtop.createHostCopy()(1) > .7); + + // Case 4: test random overlap (if non-contiguous cloudy layers, then + // random) + yakl::fortran::parallel_for( + 1, YAKL_LAMBDA(int /* dummy */) { + cldfrac_tot(1, 5) = 0.0; + cldfrac_tot(1, 6) = 0.1; + }); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(cldfrac_tot_at_cldtop.createHostCopy()(1) > + .7); // larger than the max + + // Case 5a: test independence of ice and liquid fractions + yakl::fortran::parallel_for( + 1, YAKL_LAMBDA(int /* dummy */) { + cldfrac_tot(1, 2) = 1.0; + cldfrac_tot(1, 7) = 1.0; + cldfrac_tot(1, 8) = 0.2; + }); + memset(qc, 1.0); + memset(qi, 0.0); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(cldfrac_tot_at_cldtop.createHostCopy()(1) == 1.0); + REQUIRE(cldfrac_liq_at_cldtop.createHostCopy()(1) == 1.0); + REQUIRE(cldfrac_ice_at_cldtop.createHostCopy()(1) == 0.0); + + // Case 5b: test independence of ice and liquid fractions + yakl::fortran::parallel_for( + 1, YAKL_LAMBDA(int /* dummy */) { + cldfrac_tot(1, 2) = 1.0; + cldfrac_tot(1, 7) = 1.0; + cldfrac_tot(1, 8) = 0.2; + }); + memset(qc, 0.0); + memset(qi, 1.0); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(cldfrac_tot_at_cldtop.createHostCopy()(1) == 1.0); + REQUIRE(cldfrac_liq_at_cldtop.createHostCopy()(1) == 0.0); + REQUIRE(cldfrac_ice_at_cldtop.createHostCopy()(1) == 1.0); + + // Case 6: test independence of ice and liquid fractions + // There is NOT complete independence... + // Essentially, higher ice clouds mask lower liquid clouds + // This can be problematic if the ice clouds are thin... + // We will revisit and validate this assumption later + memset(cldfrac_tot, 0.0); + memset(qc, 0.0); + memset(qi, 0.0); + yakl::fortran::parallel_for( + 1, YAKL_LAMBDA(int /* dummy */) { + cldfrac_tot(1, 2) = 0.5; // ice + cldfrac_tot(1, 3) = 0.7; // ice ------> max + cldfrac_tot(1, 4) = 0.3; // ice + // note cldfrac_tot(1, 5) is 0 + cldfrac_tot(1, 6) = 0.2; // liq + cldfrac_tot(1, 7) = 0.5; // liq ------> not max + cldfrac_tot(1, 8) = 0.1; // liq + // note cldfrac_tot(1, 9) is 0 + qi(1, 2) = 100; + qi(1, 3) = 200; + qi(1, 4) = 50; + // note qc(1, 5) is 0 + // note qi(1, 5) is 0 + qc(1, 6) = 20; + qc(1, 7) = 50; + qc(1, 8) = 10; + }); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + REQUIRE(cldfrac_tot_at_cldtop.createHostCopy()(1) > 0.70); // unaffected + REQUIRE(cldfrac_liq_at_cldtop.createHostCopy()(1) < 0.50); // not max + REQUIRE(cldfrac_ice_at_cldtop.createHostCopy()(1) == 0.7); // max + + // cleanup + tmid.deallocate(); + pmid.deallocate(); + p_del.deallocate(); + z_del.deallocate(); + qc.deallocate(); + qi.deallocate(); + rel.deallocate(); + rei.deallocate(); + cldfrac_tot.deallocate(); + nc.deallocate(); + + tmid_at_cldtop.deallocate(); + pmid_at_cldtop.deallocate(); + cldfrac_ice_at_cldtop.deallocate(); + cldfrac_liq_at_cldtop.deallocate(); + cldfrac_tot_at_cldtop.deallocate(); + cdnc_at_cldtop.deallocate(); + eff_radius_qc_at_cldtop.deallocate(); + eff_radius_qi_at_cldtop.deallocate(); + + yakl::finalize(); +} diff --git a/components/eamxx/src/physics/share/CMakeLists.txt b/components/eamxx/src/physics/share/CMakeLists.txt index a0dd7c6b9204..98e1336c8cb4 100644 --- a/components/eamxx/src/physics/share/CMakeLists.txt +++ b/components/eamxx/src/physics/share/CMakeLists.txt @@ -21,6 +21,7 @@ set_target_properties(physics_share PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules ) target_include_directories(physics_share PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/modules ) target_link_libraries(physics_share scream_share) diff --git a/components/eamxx/src/physics/share/physics_constants.hpp b/components/eamxx/src/physics/share/physics_constants.hpp index 8efbacf3113c..16750a791e3e 100644 --- a/components/eamxx/src/physics/share/physics_constants.hpp +++ b/components/eamxx/src/physics/share/physics_constants.hpp @@ -12,7 +12,7 @@ namespace scream { namespace physics { /* - * Mathematical constants used by p3. + * Mathematical constants used by atmosphere processes. * * Note that a potential optimization could be to change the type of * Scalar constants that have integer values to int. @@ -94,7 +94,6 @@ struct Constants static constexpr Scalar MWWV = MWH2O; static constexpr Scalar RWV = Rgas / MWWV; static constexpr Scalar ZVIR = (RWV / Rair) - 1.0; - static constexpr Scalar max_total_ni = 500.e+3; // maximum total ice concentration (sum of all categories) (m) static constexpr Scalar f1r = 0.78; static constexpr Scalar f2r = 0.32; static constexpr Scalar nmltratio = 1.0; // ratio of rain number produced to ice number loss from melting @@ -107,6 +106,10 @@ struct Constants static constexpr int VTABLE_DIM1 = 10; static constexpr int MU_R_TABLE_DIM = 150; + // Turbulent Mountain Stress constants + static constexpr Scalar orocnst = 1; // Converts from standard deviation to height [ no unit ] + static constexpr Scalar z0fac = 0.075; // Factor determining z_0 from orographic standard deviation [ no unit ] + // switch for warm-rain parameterization // = 1 Seifert and Beheng 2001 // = 2 Beheng 1994 @@ -117,10 +120,10 @@ struct Constants static Scalar get_gas_mol_weight(ci_string gas_name); // For use in converting area to length for a column cell - // World Geodetic System 1984 (WGS84) - static constexpr Scalar earth_ellipsoid1 = 111132.92; // first coefficient, meters per degree longitude at equator - static constexpr Scalar earth_ellipsoid2 = 559.82; // second expansion coefficient for WGS84 ellipsoid - static constexpr Scalar earth_ellipsoid3 = 1.175; // third expansion coefficient for WGS84 ellipsoid + // World Geodetic System 1984 (WGS84) + static constexpr Scalar earth_ellipsoid1 = 111132.92; // first coefficient, meters per degree longitude at equator + static constexpr Scalar earth_ellipsoid2 = 559.82; // second expansion coefficient for WGS84 ellipsoid + static constexpr Scalar earth_ellipsoid3 = 1.175; // third expansion coefficient for WGS84 ellipsoid }; // Gases diff --git a/components/eamxx/src/physics/share/physics_functions.hpp b/components/eamxx/src/physics/share/physics_functions.hpp index 155dcb112ac8..24ab66ecc5fb 100644 --- a/components/eamxx/src/physics/share/physics_functions.hpp +++ b/components/eamxx/src/physics/share/physics_functions.hpp @@ -93,10 +93,15 @@ struct Functions static Spack MurphyKoop_svp(const Spack& t, const bool ice, const Smask& range_mask, const char* caller=nullptr); // Calls a function to obtain the saturation vapor pressure, and then computes - // and returns the saturation mixing ratio, with respect to either liquid or ice, + // and returns the dry saturation mixing ratio, with respect to either liquid or ice, // depending on value of 'ice' KOKKOS_FUNCTION - static Spack qv_sat(const Spack& t_atm, const Spack& p_atm, const bool ice, const Smask& range_mask, const SaturationFcn func_idx = MurphyKoop, const char* caller=nullptr); + static Spack qv_sat_dry(const Spack& t_atm, const Spack& p_atm, const bool ice, const Smask& range_mask, const SaturationFcn func_idx = MurphyKoop, const char* caller=nullptr); + + // Calls qv_sat_dry and converts it to wet mixing ratio + KOKKOS_FUNCTION + static Spack qv_sat_wet(const Spack& t_atm, const Spack& p_atm, const bool ice, const Smask& range_mask, const Spack& dp_wet, const Spack& dp_dry, + const SaturationFcn func_idx = MurphyKoop, const char* caller=nullptr); //checks temperature for negatives and NaNs KOKKOS_FUNCTION diff --git a/components/eamxx/src/physics/share/physics_saturation_impl.hpp b/components/eamxx/src/physics/share/physics_saturation_impl.hpp index 8edffd9818df..e5707fa25cf2 100644 --- a/components/eamxx/src/physics/share/physics_saturation_impl.hpp +++ b/components/eamxx/src/physics/share/physics_saturation_impl.hpp @@ -127,11 +127,11 @@ Functions::polysvp1(const Spack& t, const bool ice, const Smask& range_mask template KOKKOS_FUNCTION typename Functions::Spack -Functions::qv_sat(const Spack& t_atm, const Spack& p_atm, const bool ice, const Smask& range_mask, const SaturationFcn func_idx, const char* caller) +Functions::qv_sat_dry(const Spack& t_atm, const Spack& p_atm_dry, const bool ice, const Smask& range_mask, const SaturationFcn func_idx, const char* caller) { /*Arguments: ---------- - t_atm: temperature; p_atm: pressure; ice: logical for ice + t_atm: temperature; p_atm_dry: dry pressure; ice: logical for ice range_mask: is a mask which masks out padded values in the packs, which are uninitialized func_idx is an optional argument to decide which scheme is to be called for saturation vapor pressure @@ -149,13 +149,38 @@ Functions::qv_sat(const Spack& t_atm, const Spack& p_atm, const bool ice, c e_pres = MurphyKoop_svp(t_atm, ice, range_mask, caller); break; default: - EKAT_KERNEL_ERROR_MSG("Error! Invalid func_idx supplied to qv_sat."); + EKAT_KERNEL_ERROR_MSG("Error! Invalid func_idx supplied to qv_sat_dry."); } static constexpr auto ep_2 = C::ep_2; - return ep_2 * e_pres / max(p_atm-e_pres, sp(1.e-3)); + return ep_2 * e_pres / max(p_atm_dry, sp(1.e-3)); } +template +KOKKOS_FUNCTION +typename Functions::Spack +Functions::qv_sat_wet(const Spack& t_atm, const Spack& p_atm_dry, const bool ice, const Smask& range_mask, + const Spack& dp_wet, const Spack& dp_dry, const SaturationFcn func_idx, const char* caller) +{ + /*Arguments (the same as in qv_sat_dry plus dp wet and dp dry): + ---------- + t_atm: temperature; p_atm_dry: dry pressure; ice: logical for ice + + range_mask: is a mask which masks out padded values in the packs, which are uninitialized + func_idx is an optional argument to decide which scheme is to be called for saturation vapor pressure + Currently default is set to "MurphyKoop_svp" + func_idx = Polysvp1 (=0) --> polysvp1 (Flatau et al. 1992) + func_idx = MurphyKoop (=1) --> MurphyKoop_svp (Murphy, D. M., and T. Koop 2005) + dp_wet: pseudo_density + dp_dry: pseudo_density_dry */ + + Spack qsatdry = qv_sat_dry(t_atm, p_atm_dry, ice, range_mask, func_idx, caller); + + return qsatdry * dp_dry / dp_wet; +} + + + } // namespace physics } // namespace scream diff --git a/components/eamxx/src/physics/share/physics_share.cpp b/components/eamxx/src/physics/share/physics_share.cpp index 263bbd9d2699..28b9ae87238a 100644 --- a/components/eamxx/src/physics/share/physics_share.cpp +++ b/components/eamxx/src/physics/share/physics_share.cpp @@ -22,7 +22,7 @@ struct CudaWrap using Scalar = ScalarT; using RangePolicy = typename ekat::KokkosTypes::RangePolicy; - static Scalar cxx_pow(Scalar base, Scalar exp) + static Scalar pow(Scalar base, Scalar exp) { Scalar result; RangePolicy policy(0,1); @@ -43,106 +43,106 @@ static Scalar wrap_name(Scalar input) { \ return result; \ } - cuda_wrap_single_arg(cxx_gamma, std::tgamma) - cuda_wrap_single_arg(cxx_sqrt, std::sqrt) - cuda_wrap_single_arg(cxx_cbrt, std::cbrt) - cuda_wrap_single_arg(cxx_log, std::log) - cuda_wrap_single_arg(cxx_log10, std::log10) - cuda_wrap_single_arg(cxx_exp, std::exp) - cuda_wrap_single_arg(cxx_expm1, std::expm1) - cuda_wrap_single_arg(cxx_tanh, std::tanh) - cuda_wrap_single_arg(cxx_erf, std::erf) + cuda_wrap_single_arg(gamma, std::tgamma) + cuda_wrap_single_arg(sqrt, std::sqrt) + cuda_wrap_single_arg(cbrt, std::cbrt) + cuda_wrap_single_arg(log, std::log) + cuda_wrap_single_arg(log10, std::log10) + cuda_wrap_single_arg(exp, std::exp) + cuda_wrap_single_arg(expm1, std::expm1) + cuda_wrap_single_arg(tanh, std::tanh) + cuda_wrap_single_arg(erf, std::erf) #undef cuda_wrap_single_arg }; extern "C" { -Real cxx_pow(Real base, Real exp) +Real scream_pow(Real base, Real exp) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_pow(base, exp); + return CudaWrap::pow(base, exp); #else return std::pow(base, exp); #endif } -Real cxx_gamma(Real input) +Real scream_gamma(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_gamma(input); + return CudaWrap::gamma(input); #else return std::tgamma(input); #endif } -Real cxx_cbrt(Real input) +Real scream_cbrt(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_cbrt(input); + return CudaWrap::cbrt(input); #else return std::cbrt(input); #endif } -Real cxx_sqrt(Real input) +Real scream_sqrt(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_sqrt(input); + return CudaWrap::sqrt(input); #else return std::sqrt(input); #endif } -Real cxx_log(Real input) +Real scream_log(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_log(input); + return CudaWrap::log(input); #else return std::log(input); #endif } -Real cxx_log10(Real input) +Real scream_log10(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_log10(input); + return CudaWrap::log10(input); #else return std::log10(input); #endif } -Real cxx_exp(Real input) +Real scream_exp(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_exp(input); + return CudaWrap::exp(input); #else return std::exp(input); #endif } -Real cxx_expm1(Real input) +Real scream_expm1(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_expm1(input); + return CudaWrap::expm1(input); #else return std::expm1(input); #endif } -Real cxx_tanh(Real input) +Real scream_tanh(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_tanh(input); + return CudaWrap::tanh(input); #else return std::tanh(input); #endif } -Real cxx_erf(Real input) +Real scream_erf(Real input) { #ifdef EAMXX_ENABLE_GPU - return CudaWrap::cxx_erf(input); + return CudaWrap::erf(input); #else return std::erf(input); #endif diff --git a/components/eamxx/src/physics/share/physics_share.hpp b/components/eamxx/src/physics/share/physics_share.hpp index 1baf06f0886b..ae61281cf0a4 100644 --- a/components/eamxx/src/physics/share/physics_share.hpp +++ b/components/eamxx/src/physics/share/physics_share.hpp @@ -7,16 +7,16 @@ namespace scream { extern "C" { -Real cxx_pow(Real base, Real exp); -Real cxx_sqrt(Real base); -Real cxx_cbrt(Real base); -Real cxx_gamma(Real input); -Real cxx_log(Real input); -Real cxx_log10(Real input); -Real cxx_exp(Real input); -Real cxx_expm1(Real input); -Real cxx_tanh(Real input); -Real cxx_erf(Real input); +Real scream_pow(Real base, Real exp); +Real scream_sqrt(Real base); +Real scream_cbrt(Real base); +Real scream_gamma(Real input); +Real scream_log(Real input); +Real scream_log10(Real input); +Real scream_exp(Real input); +Real scream_expm1(Real input); +Real scream_tanh(Real input); +Real scream_erf(Real input); } diff --git a/components/eamxx/src/physics/share/physics_share_f2c.F90 b/components/eamxx/src/physics/share/physics_share_f2c.F90 index 1737770d2ea5..1bd3a7d21d72 100644 --- a/components/eamxx/src/physics/share/physics_share_f2c.F90 +++ b/components/eamxx/src/physics/share/physics_share_f2c.F90 @@ -21,7 +21,7 @@ module physics_share_f2c ! the C++ versions in order to stay BFB. ! - function cxx_pow(base, exp) bind(C) + function scream_pow(base, exp) bind(C) use iso_c_binding !arguments: @@ -29,98 +29,98 @@ function cxx_pow(base, exp) bind(C) real(kind=c_real), value, intent(in) :: exp ! return - real(kind=c_real) :: cxx_pow - end function cxx_pow + real(kind=c_real) :: scream_pow + end function scream_pow - function cxx_sqrt(base) bind(C) + function scream_sqrt(base) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: base ! return - real(kind=c_real) :: cxx_sqrt - end function cxx_sqrt + real(kind=c_real) :: scream_sqrt + end function scream_sqrt - function cxx_cbrt(base) bind(C) + function scream_cbrt(base) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: base ! return - real(kind=c_real) :: cxx_cbrt - end function cxx_cbrt + real(kind=c_real) :: scream_cbrt + end function scream_cbrt - function cxx_gamma(input) bind(C) + function scream_gamma(input) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: input ! return - real(kind=c_real) :: cxx_gamma - end function cxx_gamma + real(kind=c_real) :: scream_gamma + end function scream_gamma - function cxx_log(input) bind(C) + function scream_log(input) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: input ! return - real(kind=c_real) :: cxx_log - end function cxx_log + real(kind=c_real) :: scream_log + end function scream_log - function cxx_log10(input) bind(C) + function scream_log10(input) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: input ! return - real(kind=c_real) :: cxx_log10 - end function cxx_log10 + real(kind=c_real) :: scream_log10 + end function scream_log10 - function cxx_exp(input) bind(C) + function scream_exp(input) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: input ! return - real(kind=c_real) :: cxx_exp - end function cxx_exp + real(kind=c_real) :: scream_exp + end function scream_exp - function cxx_expm1(input) bind(C) + function scream_expm1(input) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: input ! return - real(kind=c_real) :: cxx_expm1 - end function cxx_expm1 + real(kind=c_real) :: scream_expm1 + end function scream_expm1 - function cxx_tanh(input) bind(C) + function scream_tanh(input) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: input ! return - real(kind=c_real) :: cxx_tanh - end function cxx_tanh + real(kind=c_real) :: scream_tanh + end function scream_tanh - function cxx_erf(input) bind(C) + function scream_erf(input) bind(C) use iso_c_binding !arguments: real(kind=c_real), value, intent(in) :: input ! return - real(kind=c_real) :: cxx_erf - end function cxx_erf + real(kind=c_real) :: scream_erf + end function scream_erf end interface diff --git a/components/eamxx/src/physics/share/physics_test_data.hpp b/components/eamxx/src/physics/share/physics_test_data.hpp index e20996095cd6..099480b44393 100644 --- a/components/eamxx/src/physics/share/physics_test_data.hpp +++ b/components/eamxx/src/physics/share/physics_test_data.hpp @@ -39,7 +39,7 @@ struct SHOCGridData : public PhysicsTestData { }; */ -// Convenience macros for up to 11 arguments, beyond that, you're on your own :) +// Convenience macros for up to 20 arguments, beyond that, you're on your own :) #define PTD_ONES0 #define PTD_ONES1 1 @@ -53,6 +53,15 @@ struct SHOCGridData : public PhysicsTestData { #define PTD_ONES9 PTD_ONES8, 1 #define PTD_ONES10 PTD_ONES9, 1 #define PTD_ONES11 PTD_ONES10, 1 +#define PTD_ONES12 PTD_ONES11, 1 +#define PTD_ONES13 PTD_ONES12, 1 +#define PTD_ONES14 PTD_ONES13, 1 +#define PTD_ONES15 PTD_ONES14, 1 +#define PTD_ONES16 PTD_ONES15, 1 +#define PTD_ONES17 PTD_ONES16, 1 +#define PTD_ONES18 PTD_ONES17, 1 +#define PTD_ONES19 PTD_ONES18, 1 +#define PTD_ONES20 PTD_ONES19, 1 #define PTD_ONES(a) PTD_ONES##a @@ -70,7 +79,16 @@ struct SHOCGridData : public PhysicsTestData { #define PTD_ASS8(a, b, c, d, e, f, g, h ) PTD_ASS7(a, b, c, d, e, f, g) ; h = rhs.h #define PTD_ASS9(a, b, c, d, e, f, g, h, i ) PTD_ASS8(a, b, c, d, e, f, g, h) ; i = rhs.i #define PTD_ASS10(a, b, c, d, e, f, g, h, i, j ) PTD_ASS9(a, b, c, d, e, f, g, h, i) ; j = rhs.j -#define PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k ) PTD_ASS10(a, b, c, d, e, f, g, h, i, j) ; k = rhs.k +#define PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k ) PTD_ASS10(a, b, c, d, e, f, g, h, i, j) ; k = rhs.k +#define PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l ) PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k) ; l = rhs.l +#define PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m ) PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l) ; m = rhs.m +#define PTD_ASS14(a, b, c, d, e, f, g, h, i, j, k, l, m, n ) PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m) ; n = rhs.n +#define PTD_ASS15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o ) PTD_ASS14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) ; o = rhs.o +#define PTD_ASS16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p ) PTD_ASS15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) ; p = rhs.p +#define PTD_ASS17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q ) PTD_ASS16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) ; q = rhs.q +#define PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r ) PTD_ASS17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) ; r = rhs.r +#define PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s ) PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) ; s = rhs.s +#define PTD_ASS20(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t ) PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) ; t = rhs.t #define PTD_ASSIGN_OP(name, num_scalars, ...) \ name& operator=(const name& rhs) { PTD_ASS##num_scalars(__VA_ARGS__); assignment_impl(rhs); return *this; } diff --git a/components/eamxx/src/physics/share/scream_trcmix.cpp b/components/eamxx/src/physics/share/scream_trcmix.cpp index e53aa2b59446..fa4e8c4785d1 100644 --- a/components/eamxx/src/physics/share/scream_trcmix.cpp +++ b/components/eamxx/src/physics/share/scream_trcmix.cpp @@ -14,6 +14,7 @@ namespace physics { void trcmix( const std::string& name, + const int nlevs, trcmix_view1d const& clat, // latitude for columns in degrees trcmix_view2d const& pmid, // model pressures trcmix_view2d & q, // constituent mass mixing ratio (output) @@ -24,14 +25,10 @@ void trcmix( using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; + // Get ncol and check view bounds. const auto ncols = clat.extent(0); - const auto nlevs = pmid.extent(1); - - // - // View bounds checking. pmid and q might be padded if simd vector size > 1 - // EKAT_REQUIRE(pmid.extent(0) == ncols); - EKAT_REQUIRE(q.extent(0) == ncols && q.extent(1) == nlevs); + EKAT_REQUIRE(q.extent(0) == ncols); const auto mwn2o = C::get_gas_mol_weight("n2o"); const auto mwch4 = C::get_gas_mol_weight("ch4"); diff --git a/components/eamxx/src/physics/share/scream_trcmix.hpp b/components/eamxx/src/physics/share/scream_trcmix.hpp index d2a8c078de7f..b0fd0fd0921e 100644 --- a/components/eamxx/src/physics/share/scream_trcmix.hpp +++ b/components/eamxx/src/physics/share/scream_trcmix.hpp @@ -16,8 +16,10 @@ using trcmix_view1d = typename ekat::KokkosTypes::template view_1 template using trcmix_view2d = typename ekat::KokkosTypes::template view_2d; +// NOTE: nlevs be smaller than pmid/q extent, in case one/both of them are padded void trcmix( const std::string& name, // constituent name + const int nlevs, // number of physical levels trcmix_view1d const& clat, // latitude for columns in degrees trcmix_view2d const& pmid, // model pressures trcmix_view2d & q, // constituent mass mixing ratio (output) diff --git a/components/eamxx/src/physics/share/tests/CMakeLists.txt b/components/eamxx/src/physics/share/tests/CMakeLists.txt index 9a270577e580..f3849b7571bf 100644 --- a/components/eamxx/src/physics/share/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/share/tests/CMakeLists.txt @@ -1,10 +1,9 @@ include(ScreamUtils) -set(NEED_LIBS physics_share scream_share) - # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT ${SCREAM_BASELINES_ONLY}) - CreateUnitTest(physics_test_data physics_test_data_unit_tests.cpp "${NEED_LIBS}" +if (NOT SCREAM_BASELINES_ONLY) + CreateUnitTest(physics_test_data physics_test_data_unit_tests.cpp + LIBS physics_share THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) endif() @@ -13,8 +12,8 @@ if (SCREAM_ENABLE_BASELINE_TESTS) # The comparison test. Expects baseline to exist. All thread configurations # will use the same baseline. - CreateUnitTest( - physics_saturation_run_and_cmp "physics_saturation_run_and_cmp.cpp" "${NEED_LIBS}" + CreateUnitTest(physics_saturation_run_and_cmp "physics_saturation_run_and_cmp.cpp" + LIBS physics_share EXE_ARGS "${BASELINE_FILE_ARG}" LABELS "physics") @@ -37,5 +36,4 @@ if (SCREAM_ENABLE_BASELINE_TESTS) COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${PHYSICS_SATURATION_GEN_ARGS}) add_dependencies(baseline physics_saturation_baseline) - endif() diff --git a/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp b/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp index 630247cbabd1..cbf9945d1445 100644 --- a/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp +++ b/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp @@ -50,15 +50,17 @@ struct UnitWrap::UnitTest::TestSaturation sat_ice_fp = physics::polysvp1(temps, true, Smask(true))[0]; sat_liq_fp = physics::polysvp1(temps, false, Smask(true))[0]; - mix_ice_fr = physics::qv_sat(temps, pres, true, Smask(true), physics::Polysvp1)[0]; - mix_liq_fr = physics::qv_sat(temps, pres, false,Smask(true), physics::Polysvp1)[0]; + //Functions::qv_sat_dry(const Spack& t_atm, const Spack& p_atm_dry, const bool ice, const Smask& range_mask, + // const SaturationFcn func_idx, const char* caller) + mix_ice_fr = physics::qv_sat_dry(temps, pres, true, Smask(true), physics::Polysvp1)[0]; + mix_liq_fr = physics::qv_sat_dry(temps, pres, false,Smask(true), physics::Polysvp1)[0]; - //Get values from MurphyKoop_svp and qv_sat (qv_sat calls MurphyKoop_svp here) to test against "expected" values + //Get values from MurphyKoop_svp and qv_sat_dry (qv_sat_dry calls MurphyKoop_svp here) to test against "expected" values sat_ice_mkp = physics::MurphyKoop_svp(temps, true, Smask(true))[0]; sat_liq_mkp = physics::MurphyKoop_svp(temps, false, Smask(true))[0]; - mix_ice_mkr = physics::qv_sat(temps, pres, true, Smask(true), physics::MurphyKoop)[0]; - mix_liq_mkr = physics::qv_sat(temps, pres, false, Smask(true), physics::MurphyKoop)[0]; + mix_ice_mkr = physics::qv_sat_dry(temps, pres, true, Smask(true), physics::MurphyKoop)[0]; + mix_liq_mkr = physics::qv_sat_dry(temps, pres, false, Smask(true), physics::MurphyKoop)[0]; } static constexpr auto atm_pres = 1e5; diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 52955492d902..051f9635d179 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -4,7 +4,7 @@ set(SHOC_SRCS shoc_iso_c.f90 shoc_iso_f.f90 ${SCREAM_BASE_DIR}/../eam/src/physics/cam/shoc.F90 - atmosphere_macrophysics.cpp + eamxx_shoc_process_interface.cpp ) if (NOT SCREAM_LIB_ONLY) @@ -16,7 +16,7 @@ endif() set(SHOC_HEADERS shoc.hpp - atmosphere_macrophysics.hpp + eamxx_shoc_process_interface.hpp shoc_constants.hpp ) @@ -35,6 +35,7 @@ if (NOT EAMXX_ENABLE_GPU) eti/shoc_compute_l_inf_shoc_length.cpp eti/shoc_compute_shoc_mix_shoc_length.cpp eti/shoc_compute_shoc_vapor.cpp + eti/shoc_compute_shoc_temperature.cpp eti/shoc_compute_shr_prod.cpp eti/shoc_compute_tmpi.cpp eti/shoc_diag_obklen.cpp @@ -74,6 +75,7 @@ set(SHOC_SK_SRCS disp/shoc_check_tke_disp.cpp disp/shoc_grid_disp.cpp disp/shoc_compute_shoc_vapor_disp.cpp + disp/shoc_compute_shoc_temperature_disp.cpp disp/shoc_diag_obklen_disp.cpp disp/shoc_pblintd_disp.cpp disp/shoc_length_disp.cpp @@ -113,13 +115,13 @@ else() list(APPEND SHOC_LIBS "shoc_sk") endif() endif() +target_compile_definitions(shoc PUBLIC EAMXX_HAS_SHOC) foreach (SHOC_LIB IN LISTS SHOC_LIBS) set_target_properties(${SHOC_LIB} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${SHOC_LIB}_modules ) target_include_directories(${SHOC_LIB} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../share ${CMAKE_CURRENT_BINARY_DIR}/${SHOC_LIB}_modules ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/impl diff --git a/components/eamxx/src/physics/shoc/disp/shoc_compute_shoc_temperature_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_compute_shoc_temperature_disp.cpp new file mode 100644 index 000000000000..22c5e918826e --- /dev/null +++ b/components/eamxx/src/physics/shoc/disp/shoc_compute_shoc_temperature_disp.cpp @@ -0,0 +1,34 @@ +#include "shoc_functions.hpp" + +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace shoc { + +template<> +void Functions +::compute_shoc_temperature_disp( + const Int& shcol, + const Int& nlev, + const view_2d& thetal, + const view_2d& ql, + const view_2d& inv_exner, + const view_2d& tabs) +{ + using ExeSpace = typename KT::ExeSpace; + + const auto nlev_packs = ekat::npack(nlev); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nlev_packs); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const Int i = team.league_rank(); + + compute_shoc_temperature(team, nlev, + ekat::subview(thetal, i), + ekat::subview(ql, i), + ekat::subview(inv_exner, i), + ekat::subview(tabs, i)); + }); +} + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/disp/shoc_diag_second_shoc_moments_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_diag_second_shoc_moments_disp.cpp index 22a5ad8ef6f9..5561b80324d2 100644 --- a/components/eamxx/src/physics/shoc/disp/shoc_diag_second_shoc_moments_disp.cpp +++ b/components/eamxx/src/physics/shoc/disp/shoc_diag_second_shoc_moments_disp.cpp @@ -9,6 +9,7 @@ template<> void Functions ::diag_second_shoc_moments_disp( const Int& shcol, const Int& nlev, const Int& nlevi, + const Real& thl2tune, const Real& qw2tune, const Real& qwthl2tune, const Real& w2tune, const view_2d& thetal, const view_2d& qw, const view_2d& u_wind, @@ -49,6 +50,7 @@ ::diag_second_shoc_moments_disp( diag_second_shoc_moments( team, nlev, nlevi, + thl2tune, qw2tune, qwthl2tune, w2tune, ekat::subview(thetal, i), ekat::subview(qw, i), ekat::subview(u_wind, i), diff --git a/components/eamxx/src/physics/shoc/disp/shoc_diag_third_shoc_moments_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_diag_third_shoc_moments_disp.cpp index f9e025e3b9ab..9d9ac106a3fb 100644 --- a/components/eamxx/src/physics/shoc/disp/shoc_diag_third_shoc_moments_disp.cpp +++ b/components/eamxx/src/physics/shoc/disp/shoc_diag_third_shoc_moments_disp.cpp @@ -11,6 +11,7 @@ ::diag_third_shoc_moments_disp( const Int& shcol, const Int& nlev, const Int& nlevi, + const Scalar& c_diag_3rd_mom, const view_2d& w_sec, const view_2d& thl_sec, const view_2d& wthl_sec, @@ -36,6 +37,7 @@ ::diag_third_shoc_moments_disp( diag_third_shoc_moments( team, nlev, nlevi, + c_diag_3rd_mom, ekat::subview(w_sec, i), ekat::subview(thl_sec, i), ekat::subview(wthl_sec, i), diff --git a/components/eamxx/src/physics/shoc/disp/shoc_length_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_length_disp.cpp index c826b1414ca7..a30078816548 100644 --- a/components/eamxx/src/physics/shoc/disp/shoc_length_disp.cpp +++ b/components/eamxx/src/physics/shoc/disp/shoc_length_disp.cpp @@ -11,6 +11,7 @@ ::shoc_length_disp( const Int& shcol, const Int& nlev, const Int& nlevi, + const Scalar& length_fac, const view_1d& dx, const view_1d& dy, const view_2d& zt_grid, @@ -31,7 +32,8 @@ ::shoc_length_disp( auto workspace = workspace_mgr.get_workspace(team); - shoc_length(team, nlev, nlevi, dx(i), dy(i), + shoc_length(team, nlev, nlevi, length_fac, + dx(i), dy(i), ekat::subview(zt_grid, i), ekat::subview(zi_grid, i), ekat::subview(dz_zt, i), diff --git a/components/eamxx/src/physics/shoc/disp/shoc_tke_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_tke_disp.cpp index 6b847255ee3a..eb66cd2c4431 100644 --- a/components/eamxx/src/physics/shoc/disp/shoc_tke_disp.cpp +++ b/components/eamxx/src/physics/shoc/disp/shoc_tke_disp.cpp @@ -12,15 +12,21 @@ ::shoc_tke_disp( const Int& nlev, const Int& nlevi, const Scalar& dtime, + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, + const Scalar& Ckh, + const Scalar& Ckm, const view_2d& wthv_sec, const view_2d& shoc_mix, const view_2d& dz_zi, const view_2d& dz_zt, const view_2d& pres, + const view_2d& tabs, const view_2d& u_wind, const view_2d& v_wind, const view_2d& brunt, - const view_1d& obklen, const view_2d& zt_grid, const view_2d& zi_grid, const view_1d& pblh, @@ -40,15 +46,17 @@ ::shoc_tke_disp( auto workspace = workspace_mgr.get_workspace(team); shoc_tke(team, nlev, nlevi, dtime, + lambda_low, lambda_high, lambda_slope, lambda_thresh, + Ckh, Ckm, ekat::subview(wthv_sec, i), ekat::subview(shoc_mix, i), ekat::subview(dz_zi, i), ekat::subview(dz_zt, i), ekat::subview(pres, i), + ekat::subview(tabs, i), ekat::subview(u_wind, i), ekat::subview(v_wind, i), ekat::subview(brunt, i), - obklen(i), ekat::subview(zt_grid, i), ekat::subview(zi_grid, i), pblh(i), diff --git a/components/eamxx/src/physics/shoc/atmosphere_macrophysics.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp similarity index 69% rename from components/eamxx/src/physics/shoc/atmosphere_macrophysics.cpp rename to components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 351c54a9823a..c107e53b5b31 100644 --- a/components/eamxx/src/physics/shoc/atmosphere_macrophysics.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -1,5 +1,5 @@ #include "ekat/ekat_assert.hpp" -#include "physics/shoc/atmosphere_macrophysics.hpp" +#include "physics/shoc/eamxx_shoc_process_interface.hpp" #include "share/property_checks/field_lower_bound_check.hpp" #include "share/property_checks/field_within_interval_check.hpp" @@ -35,9 +35,6 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column - m_cell_area = m_grid->get_geometry_data("area").get_view(); // area of each cell - m_cell_lat = m_grid->get_geometry_data("lat").get_view(); // area of each cell - // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -45,7 +42,7 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids FieldLayout scalar2d_layout_col{ {COL}, {m_num_cols} }; // Layout for surf_mom_flux - FieldLayout surf_mom_flux_layout { {COL, CMP}, {m_num_cols, 2} }; + FieldLayout surf_mom_flux_layout { {COL, CMP}, {m_num_cols, 2} }; // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; @@ -64,13 +61,18 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids const auto s2 = s*s; // These variables are needed by the interface, but not actually passed to shoc_main. - add_field("omega", scalar3d_layout_mid, Pa/s, grid_name, ps); - add_field("surf_sens_flux", scalar2d_layout_col, W/m2, grid_name); - add_field("surf_evap", scalar2d_layout_col, kg/m2/s, grid_name); - add_field("surf_mom_flux", surf_mom_flux_layout, N/m2, grid_name); + add_field("omega", scalar3d_layout_mid, Pa/s, grid_name, ps); + add_field("surf_sens_flux", scalar2d_layout_col, W/m2, grid_name); + add_field("surf_mom_flux", surf_mom_flux_layout, N/m2, grid_name); + + add_field("surf_evap", scalar2d_layout_col, kg/m2/s, grid_name); + add_field ("T_mid", scalar3d_layout_mid, K, grid_name, ps); + add_field ("qv", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); - add_field ("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field ("qv", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + // If TMS is a process, add surface drag coefficient to required fields + if (m_params.get("apply_tms", false)) { + add_field("surf_drag_coeff_tms", scalar2d_layout_col, kg/s/m2, grid_name); + } // Input variables add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); @@ -91,7 +93,7 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); // Tracer group - add_group("tracers",grid_name,ps,Bundling::Required); + add_group("tracers", grid_name, ps, Bundling::Required); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { @@ -139,7 +141,7 @@ size_t SHOCMacrophysics::requested_buffer_size_in_bytes() const const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, nlev_packs); const int n_wind_slots = ekat::npack(2)*Spack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const size_t wsm_request= WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy); + const size_t wsm_request= WSM::get_total_bytes_needed(nlevi_packs, 14+(n_wind_slots+n_trac_slots), policy); return interface_request + wsm_request; } @@ -152,9 +154,9 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) Real* mem = reinterpret_cast(buffer_manager.get_memory()); // 1d scalar views - using scalar_view_t = decltype(m_buffer.cell_length); + using scalar_view_t = decltype(m_buffer.wpthlp_sfc); scalar_view_t* _1d_scalar_view_ptrs[Buffer::num_1d_scalar_ncol] = - {&m_buffer.cell_length, &m_buffer.wpthlp_sfc, &m_buffer.wprtp_sfc, &m_buffer.upwp_sfc, &m_buffer.vpwp_sfc + {&m_buffer.wpthlp_sfc, &m_buffer.wprtp_sfc, &m_buffer.upwp_sfc, &m_buffer.vpwp_sfc #ifdef SCREAM_SMALL_KERNELS , &m_buffer.se_b, &m_buffer.ke_b, &m_buffer.wv_b, &m_buffer.wl_b , &m_buffer.se_a, &m_buffer.ke_a, &m_buffer.wv_a, &m_buffer.wl_a @@ -182,7 +184,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.inv_exner, &m_buffer.thlm, &m_buffer.qw, &m_buffer.dse, &m_buffer.tke_copy, &m_buffer.qc_copy, &m_buffer.shoc_ql2, &m_buffer.shoc_mix, &m_buffer.isotropy, &m_buffer.w_sec, &m_buffer.wqls_sec, &m_buffer.brunt #ifdef SCREAM_SMALL_KERNELS - , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.dz_zt, &m_buffer.tkh + , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.tabs, &m_buffer.dz_zt, &m_buffer.tkh #endif }; @@ -215,7 +217,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, nlev_packs); const int n_wind_slots = ekat::npack(2)*Spack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); + const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 14+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); s_mem += wsm_size; size_t used_mem = (reinterpret_cast(s_mem) - buffer_manager.get_memory())*sizeof(Real); @@ -225,32 +227,43 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) // ========================================================================================= void SHOCMacrophysics::initialize_impl (const RunType run_type) { + // Gather runtime options + runtime_options.lambda_low = m_params.get("lambda_low"); + runtime_options.lambda_high = m_params.get("lambda_high"); + runtime_options.lambda_slope = m_params.get("lambda_slope"); + runtime_options.lambda_thresh = m_params.get("lambda_thresh"); + runtime_options.thl2tune = m_params.get("thl2tune"); + runtime_options.qw2tune = m_params.get("qw2tune"); + runtime_options.qwthl2tune = m_params.get("qwthl2tune"); + runtime_options.w2tune = m_params.get("w2tune"); + runtime_options.length_fac = m_params.get("length_fac"); + runtime_options.c_diag_3rd_mom = m_params.get("c_diag_3rd_mom"); + runtime_options.Ckh = m_params.get("Ckh"); + runtime_options.Ckm = m_params.get("Ckm"); // Initialize all of the structures that are passed to shoc_main in run_impl. // Note: Some variables in the structures are not stored in the field manager. For these // variables a local view is constructed. - const auto& T_mid = get_field_out("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& p_int = get_field_in("p_int").get_view(); - const auto& pseudo_density = get_field_in("pseudo_density").get_view(); - const auto& omega = get_field_in("omega").get_view(); - const auto& surf_sens_flux = get_field_in("surf_sens_flux").get_view(); - const auto& surf_evap = get_field_in("surf_evap").get_view(); - const auto& surf_mom_flux = get_field_in("surf_mom_flux").get_view(); - const auto& qtracers = get_group_out("tracers").m_bundle->get_view(); - const auto& qc = get_field_out("qc").get_view(); - const auto& qv = get_field_out("qv").get_view(); - const auto& tke = get_field_out("tke").get_view(); - const auto& cldfrac_liq = get_field_out("cldfrac_liq").get_view(); - const auto& sgs_buoy_flux = get_field_out("sgs_buoy_flux").get_view(); - const auto& tk = get_field_out("eddy_diff_mom").get_view(); - const auto& inv_qc_relvar = get_field_out("inv_qc_relvar").get_view(); - const auto& phis = get_field_in("phis").get_view(); - const auto& tracer_info = get_group_out("tracers").m_info; // obtain tracer info structure + const auto& T_mid = get_field_out("T_mid").get_view(); + const auto& p_mid = get_field_in("p_mid").get_view(); + const auto& p_int = get_field_in("p_int").get_view(); + const auto& pseudo_density = get_field_in("pseudo_density").get_view(); + const auto& omega = get_field_in("omega").get_view(); + const auto& surf_sens_flux = get_field_in("surf_sens_flux").get_view(); + const auto& surf_evap = get_field_in("surf_evap").get_view(); + const auto& surf_mom_flux = get_field_in("surf_mom_flux").get_view(); + const auto& qtracers = get_group_out("tracers").m_bundle->get_view(); + const auto& qc = get_field_out("qc").get_view(); + const auto& qv = get_field_out("qv").get_view(); + const auto& tke = get_field_out("tke").get_view(); + const auto& cldfrac_liq = get_field_out("cldfrac_liq").get_view(); + const auto& sgs_buoy_flux = get_field_out("sgs_buoy_flux").get_view(); + const auto& tk = get_field_out("eddy_diff_mom").get_view(); + const auto& inv_qc_relvar = get_field_out("inv_qc_relvar").get_view(); + const auto& phis = get_field_in("phis").get_view(); // Alias local variables from temporary buffer auto z_mid = m_buffer.z_mid; auto z_int = m_buffer.z_int; - auto cell_length = m_buffer.cell_length; auto wpthlp_sfc = m_buffer.wpthlp_sfc; auto wprtp_sfc = m_buffer.wprtp_sfc; auto upwp_sfc = m_buffer.upwp_sfc; @@ -282,40 +295,14 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) Kokkos::deep_copy(tke_copy,0.0004); Kokkos::deep_copy(cldfrac_liq,0.0); } - // Find index of qv (water vapor, kg/kg(wet-air) and tke (J/kg(wet-air)) in the qtracer 3d view - // These indices are later used for converting tracers from wet mmr to dry mmr and vice-versa - auto qv_index = tracer_info->m_subview_idx.at("qv"); - auto tke_index = tracer_info->m_subview_idx.at("tke"); - - //Device view to store indices of tracers which will participate in wet<->dry conversion; we are excluding - //"tke" [as it is not "water based" tracer] and "qv"[as "qv" (before conversion) is needed for - //computing conversion for all other tracers] from qtracers view - - view_1d_int convert_wet_dry_idx_d("convert_wet_dry_idx_d",m_num_tracers-2); //2 tracers, qv and tke, are excluded - - //mirror view on host - auto convert_wet_dry_idx_h = Kokkos::create_mirror_view(convert_wet_dry_idx_d); - //loop over all tracers to store of all tracer indices except for tke and qv - for (int it=0,iq=0; it(); const auto& heat_flux = get_field_out("heat_flux").get_view(); shoc_postprocess.set_mass_and_energy_fluxes (surf_evap, surf_sens_flux, - vapor_flux, water_flux, + vapor_flux, water_flux, ice_flux, heat_flux); } @@ -419,7 +407,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const int n_wind_slots = ekat::npack(2)*Spack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, nlev_packs); - workspace_mgr.setup(m_buffer.wsm_data, nlevi_packs, 13+(n_wind_slots+n_trac_slots), default_policy); + workspace_mgr.setup(m_buffer.wsm_data, nlevi_packs, 14+(n_wind_slots+n_trac_slots), default_policy); // Calculate pref_mid, and use that to calculate // maximum number of levels in pbl from surface @@ -437,6 +425,26 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const int ntop_shoc = 0; const int nbot_shoc = m_num_levs; m_npbl = SHF::shoc_init(nbot_shoc,ntop_shoc,pref_mid); + + // Compute cell length for input dx and dy. + const auto ncols = m_num_cols; + view_1d cell_length("cell_length", ncols); + if (m_grid->has_geometry_data("dx_short")) { + // We must be running with IntensiveObservationPeriod on, with a planar geometry + auto dx = m_grid->get_geometry_data("dx_short").get_view()(); + Kokkos::deep_copy(cell_length, dx); + } else { + const auto area = m_grid->get_geometry_data("area").get_view(); + const auto lat = m_grid->get_geometry_data("lat").get_view(); + Kokkos::parallel_for(ncols, KOKKOS_LAMBDA (const int icol) { + // For now, we are considering dy=dx. Here, we + // will need to compute dx/dy instead of cell_length + // if we have dy!=dx. + cell_length(icol) = PF::calculate_dx_from_area(area(icol),lat(icol));; + }); + } + input.dx = cell_length; + input.dy = cell_length; } // ========================================================================================= @@ -444,7 +452,7 @@ void SHOCMacrophysics::run_impl (const double dt) { EKAT_REQUIRE_MSG (dt<=300, "Error! SHOC is intended to run with a timestep no longer than 5 minutes.\n" - " Please, reduce timestep (perhaps increasing subcycling iteratinos).\n"); + " Please, reduce timestep (perhaps increasing subcycling iterations).\n"); const auto nlev_packs = ekat::npack(m_num_levs); const auto scan_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(m_num_cols, nlev_packs); @@ -457,6 +465,14 @@ void SHOCMacrophysics::run_impl (const double dt) shoc_preprocess); Kokkos::fence(); + if (m_params.get("apply_tms", false)) { + apply_turbulent_mountain_stress(); + } + + if (m_params.get("check_flux_state_consistency", false)) { + check_flux_state_consistency(dt); + } + // For now set the host timestep to the shoc timestep. This forces // number of SHOC timesteps (nadv) to be 1. // TODO: input parameter? @@ -468,7 +484,7 @@ void SHOCMacrophysics::run_impl (const double dt) // Run shoc main SHF::shoc_main(m_num_cols, m_num_levs, m_num_levs+1, m_npbl, m_nadv, m_num_tracers, dt, - workspace_mgr,input,input_output,output,history_output + workspace_mgr,runtime_options,input,input_output,output,history_output #ifdef SCREAM_SMALL_KERNELS , temporaries #endif @@ -486,5 +502,73 @@ void SHOCMacrophysics::finalize_impl() // Do nothing } // ========================================================================================= +void SHOCMacrophysics::apply_turbulent_mountain_stress() +{ + auto surf_drag_coeff_tms = get_field_in("surf_drag_coeff_tms").get_view(); + auto horiz_winds = get_field_in("horiz_winds").get_view(); + + auto rrho_i = m_buffer.rrho_i; + auto upwp_sfc = m_buffer.upwp_sfc; + auto vpwp_sfc = m_buffer.vpwp_sfc; + + const int nlev_v = (m_num_levs-1)/Spack::n; + const int nlev_p = (m_num_levs-1)%Spack::n; + const int nlevi_v = m_num_levs/Spack::n; + const int nlevi_p = m_num_levs%Spack::n; + + Kokkos::parallel_for("apply_tms", KT::RangePolicy(0, m_num_cols), KOKKOS_LAMBDA (const int i) { + upwp_sfc(i) -= surf_drag_coeff_tms(i)*horiz_winds(i,0,nlev_v)[nlev_p]/rrho_i(i,nlevi_v)[nlevi_p]; + vpwp_sfc(i) -= surf_drag_coeff_tms(i)*horiz_winds(i,1,nlev_v)[nlev_p]/rrho_i(i,nlevi_v)[nlevi_p]; + }); +} +// ========================================================================================= +void SHOCMacrophysics::check_flux_state_consistency(const double dt) +{ + using PC = scream::physics::Constants; + const Real gravit = PC::gravit; + const Real qmin = 1e-12; // minimum permitted constituent concentration (kg/kg) + + const auto& pseudo_density = get_field_in ("pseudo_density").get_view(); + const auto& surf_evap = get_field_out("surf_evap").get_view(); + const auto& qv = get_field_out("qv").get_view(); + const auto nlevs = m_num_levs; + const auto nlev_packs = ekat::npack(nlevs); + const auto last_pack_idx = (nlevs-1)/Spack::n; + const auto last_pack_entry = (nlevs-1)%Spack::n; + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, nlev_packs); + Kokkos::parallel_for("check_flux_state_consistency", + policy, + KOKKOS_LAMBDA (const KT::MemberType& team) { + const auto i = team.league_rank(); + + const auto& pseudo_density_i = ekat::subview(pseudo_density, i); + const auto& qv_i = ekat::subview(qv, i); + + // reciprocal of pseudo_density at the bottom layer + const auto rpdel = 1.0/pseudo_density_i(last_pack_idx)[last_pack_entry]; + + // Check if the negative surface latent heat flux can exhaust + // the moisture in the lowest model level. If so, apply fixer. + const auto condition = surf_evap(i) - (qmin - qv_i(last_pack_idx)[last_pack_entry])/(dt*gravit*rpdel); + if (condition < 0) { + const auto cc = abs(surf_evap(i)*dt*gravit); + + auto tracer_mass = [&](const int k) { + return qv_i(k)*pseudo_density_i(k); + }; + Real mm = ekat::ExeSpaceUtils::view_reduction(team, 0, nlevs, tracer_mass); + + EKAT_KERNEL_ASSERT_MSG(mm >= cc, "Error! Total mass of column vapor should be greater than mass of surf_evap.\n"); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&](const int& k) { + const auto adjust = cc*qv_i(k)*pseudo_density_i(k)/mm; + qv_i(k) = (qv_i(k)*pseudo_density_i(k) - adjust)/pseudo_density_i(k); + }); + + surf_evap(i) = 0; + } + }); +} +// ========================================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/shoc/atmosphere_macrophysics.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp similarity index 75% rename from components/eamxx/src/physics/shoc/atmosphere_macrophysics.hpp rename to components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 0b950bd6fa57..099794adafce 100644 --- a/components/eamxx/src/physics/shoc/atmosphere_macrophysics.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -83,24 +83,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const int nlev_packs = ekat::npack(nlev); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { - /*--------------------------------------------------------------------------------- - *Wet to dry mixing ratios: - *------------------------- - *Since tracers from the host model (or AD) are wet mixing ratios and SHOC expects - *these tracers in dry mixing ratios, we convert the wet mixing ratios to dry mixing - *ratios for all the tracers *except* for qv [which is done after all the other tracers] - *and TKE [which is always defined in units of J/(kg of total air) and therefore - *should be considered to be "wet" when stored in the FM and also "wet" when used in - *SHOC]. - *---------------------------------------------------------------------------------- - */ - //NOTE:Function calculate_drymmr_from_wetmmr takes 2 arguments: ( wet mmr and "wet" - //water vapor mixing ratio) - //Units of all tracers (except TKE and qv) will become [kg/kg(dry-air)] for mass and - //[#/kg(dry-air)] for number after the following conversion. qv will be converted - //to dry mmr in the next parallel for - for (Int iq = 0; iq < num_qtracers-2; ++iq) - qtracers(i,convert_wet_dry_idx_d(iq),k) = PF::calculate_drymmr_from_wetmmr(qtracers(i,convert_wet_dry_idx_d(iq),k), qv(i,k)); const auto range = ekat::range(k*Spack::n); const Smask in_nlev_range = (range < nlev); @@ -111,9 +93,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess EKAT_KERNEL_ASSERT((nonzero || !in_nlev_range).all()); inv_exner(i,k).set(nonzero, 1/exner); - //At this point, convert qv to dry mmr, the units will become kg/kg(dry air) - qv(i,k) = PF::calculate_drymmr_from_wetmmr(qv(i,k), qv(i,k)); - tke(i,k) = ekat::max(sp(0.004), tke(i,k)); // Tracers are updated as a group. The tracers tke and qc act as separate inputs to shoc_main() @@ -123,8 +102,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // to tracer group in postprocessing. // TODO: remove *_copy views once SHOC can request a subset of tracers. tke_copy(i,k) = tke(i,k); - qc_copy(i,k) = qc(i,k); //at this point, qc should be dry mmr [kg/kg(dry air)] - + qc_copy(i,k) = qc(i,k); qw(i,k) = qv(i,k) + qc(i,k); @@ -169,18 +147,13 @@ class SHOCMacrophysics : public scream::AtmosphereProcess SHF::linear_interp(team,zt_grid_s,zi_grid_s,rrho_s,rrho_i_s,nlev,nlev+1,0); team.team_barrier(); - // For now, we are considering dy=dx. Here, we - // will need to compute dx/dy instead of cell_length - // if we have dy!=dx. - cell_length(i) = PF::calculate_dx_from_area(area(i),lat(i)); - const auto exner_int = PF::exner_function(p_int(i,nlevi_v)[nlevi_p]); const auto inv_exner_int_surf = 1/exner_int; wpthlp_sfc(i) = (surf_sens_flux(i)/(cpair*rrho_i(i,nlevi_v)[nlevi_p]))*inv_exner_int_surf; wprtp_sfc(i) = surf_evap(i)/rrho_i(i,nlevi_v)[nlevi_p]; - upwp_sfc(i) = surf_mom_flux(i,0)/rrho_i(i,nlevi_v)[nlevi_p]; - vpwp_sfc(i) = surf_mom_flux(i,1)/rrho_i(i,nlevi_v)[nlevi_p]; + upwp_sfc(i) = surf_mom_flux(i,0)/rrho_i(i,nlevi_v)[nlevi_p]; + vpwp_sfc(i) = surf_mom_flux(i,1)/rrho_i(i,nlevi_v)[nlevi_p]; const int num_qtracer_packs = ekat::npack(num_qtracers); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, num_qtracer_packs), [&] (const Int& q) { @@ -191,50 +164,44 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Local variables int ncol, nlev, num_qtracers; Real z_surf; - view_1d_int convert_wet_dry_idx_d; - view_1d_const area; - view_1d_const lat; - view_2d_const T_mid; - view_2d_const p_mid; - view_2d_const p_int; - view_2d_const pseudo_density; - view_2d_const omega; - view_1d_const phis; - view_1d_const surf_sens_flux; - view_1d_const surf_evap; - sview_2d_const surf_mom_flux; - view_3d qtracers; - view_2d qv; - view_2d_const qc; - view_2d qc_copy; - view_2d z_mid; - view_2d z_int; - view_1d cell_length; - view_2d shoc_s; - view_2d tke; - view_2d tke_copy; - view_2d rrho; - view_2d rrho_i; - view_2d thv; - view_2d dz; - view_2d zt_grid; - view_2d zi_grid; - view_1d wpthlp_sfc; - view_1d wprtp_sfc; - view_1d upwp_sfc; - view_1d vpwp_sfc; - view_2d wtracer_sfc; - view_2d wm_zt; - view_2d inv_exner; - view_2d thlm; - view_2d qw; - view_2d cloud_frac; + view_2d_const T_mid; + view_2d_const p_mid; + view_2d_const p_int; + view_2d_const pseudo_density; + view_2d_const omega; + view_1d_const phis; + view_1d_const surf_sens_flux; + view_1d_const surf_evap; + sview_2d_const surf_mom_flux; + view_3d qtracers; + view_2d qv; + view_2d_const qc; + view_2d qc_copy; + view_2d z_mid; + view_2d z_int; + view_2d shoc_s; + view_2d tke; + view_2d tke_copy; + view_2d rrho; + view_2d rrho_i; + view_2d thv; + view_2d dz; + view_2d zt_grid; + view_2d zi_grid; + view_1d wpthlp_sfc; + view_1d wprtp_sfc; + view_1d upwp_sfc; + view_1d vpwp_sfc; + view_2d wtracer_sfc; + view_2d wm_zt; + view_2d inv_exner; + view_2d thlm; + view_2d qw; + view_2d cloud_frac; // Assigning local variables void set_variables(const int ncol_, const int nlev_, const int num_qtracers_, - const view_1d_int& convert_wet_dry_idx_d_, const Real z_surf_, - const view_1d_const& area_, const view_1d_const& lat_, const view_2d_const& T_mid_, const view_2d_const& p_mid_, const view_2d_const& p_int_, const view_2d_const& pseudo_density_, const view_2d_const& omega_, const view_1d_const& phis_, const view_1d_const& surf_sens_flux_, const view_1d_const& surf_evap_, @@ -243,7 +210,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const view_2d& qv_, const view_2d_const& qc_, const view_2d& qc_copy_, const view_2d& tke_, const view_2d& tke_copy_, const view_2d& z_mid_, const view_2d& z_int_, - const view_1d& cell_length_, const view_2d& dse_, const view_2d& rrho_, const view_2d& rrho_i_, const view_2d& thv_, const view_2d& dz_,const view_2d& zt_grid_,const view_2d& zi_grid_, const view_1d& wpthlp_sfc_, const view_1d& wprtp_sfc_,const view_1d& upwp_sfc_,const view_1d& vpwp_sfc_, const view_2d& wtracer_sfc_, @@ -252,11 +218,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess ncol = ncol_; nlev = nlev_; num_qtracers = num_qtracers_; - convert_wet_dry_idx_d = convert_wet_dry_idx_d_; z_surf = z_surf_; // IN - area = area_; - lat = lat_; T_mid = T_mid_; p_mid = p_mid_; p_int = p_int_; @@ -276,7 +239,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess tke_copy = tke_copy_; z_mid = z_mid_; z_int = z_int_; - cell_length = cell_length_; rrho = rrho_; rrho_i = rrho_i_; thv = thv_; @@ -309,9 +271,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const Real inv_qc_relvar_max = 10; const Real inv_qc_relvar_min = 0.001; - //In the following loop, tke, qc and qv are updated. All these variables are slices of "qtracers" array - //After these updates, all tracers (except TKE) in the qtarcers array will be converted - //from dry mmr to wet mmr const int nlev_packs = ekat::npack(nlev); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { // See comment in SHOCPreprocess::operator() about the necessity of *_copy views @@ -322,10 +281,11 @@ class SHOCMacrophysics : public scream::AtmosphereProcess cldfrac_liq(i,k) = ekat::min(cldfrac_liq(i,k), 1); + //P3 uses inv_qc_relvar, P3 is using dry mmrs, but + //wet<->dry conversion is a constant factor that cancels out in mean(qc)^2/mean(qc'*qc'). inv_qc_relvar(i,k) = 1; const auto condition = (qc(i,k) != 0 && qc2(i,k) != 0); if (condition.any()) { - //inv_qc_relvar is used in P3 and is computed here using qc and qc2, which are in dry mmr inv_qc_relvar(i,k).set(condition, ekat::min(inv_qc_relvar_max, ekat::max(inv_qc_relvar_min, @@ -338,25 +298,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const Real phis_i(phis(i)); T_mid(i,k) = PF::calculate_temperature_from_dse(dse_ik,z_mid_ik,phis_i); - - /*-------------------------------------------------------------------------------- - *DRY-TO-WET MMRs: - *----------------- - *Since the host model (or AD) expects wet mixing ratios, we need to convert dry - *mixing ratios from SHOC to wet mixing ratios except for qv[which will be converted - *in the following "parallel for" after all the other tracers] and TKE [which is - already in wet mmr]. - *--------------------------------------------------------------------------------- - */ - - //NOTE:Function calculate_wetmmr_from_drymmr takes 2 arguments: ( dry mmr and "dry" - //water vapor mixing ratio) - //Units of all tracers (except TKE and qv) will become [kg/kg(wet-air)] for mass and - //[#/kg(wet-air)] for number after the following conversion. qv will be converted - //to wet mmr in the next parallel for - for (Int iq = 0; iq < num_qtracers-2; ++iq) - qtracers(i,convert_wet_dry_idx_d(iq),k) = PF::calculate_wetmmr_from_drymmr(qtracers(i,convert_wet_dry_idx_d(iq),k), qv(i,k)); - qv(i,k) = PF::calculate_wetmmr_from_drymmr(qv(i,k), qv(i,k)); }); // If necessary, set appropriate boundary fluxes for energy and mass conservation checks. @@ -371,7 +312,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Local variables int ncol, nlev, num_qtracers; - view_1d_int convert_wet_dry_idx_d; view_2d_const rrho; view_2d qv, qc, tke; view_2d_const tke_copy, qc_copy, qw; @@ -392,7 +332,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Assigning local variables void set_variables(const int ncol_, const int nlev_, const int num_qtracers_, - const view_1d_int& convert_wet_dry_idx_d_, const view_2d_const& rrho_, const view_2d& qv_, const view_2d_const& qw_, const view_2d& qc_, const view_2d_const& qc_copy_, const view_2d& tke_, const view_2d_const& tke_copy_, const view_3d& qtracers_, const view_2d_const& qc2_, @@ -402,7 +341,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess ncol = ncol_; nlev = nlev_; num_qtracers = num_qtracers_; - convert_wet_dry_idx_d = convert_wet_dry_idx_d_; rrho = rrho_; qv = qv_; qw = qw_; @@ -438,21 +376,20 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Structure for storing local variables initialized using the ATMBufferManager struct Buffer { #ifndef SCREAM_SMALL_KERNELS - static constexpr int num_1d_scalar_ncol = 5; + static constexpr int num_1d_scalar_ncol = 4; #else - static constexpr int num_1d_scalar_ncol = 18; + static constexpr int num_1d_scalar_ncol = 17; #endif static constexpr int num_1d_scalar_nlev = 1; #ifndef SCREAM_SMALL_KERNELS static constexpr int num_2d_vector_mid = 18; static constexpr int num_2d_vector_int = 12; #else - static constexpr int num_2d_vector_mid = 22; + static constexpr int num_2d_vector_mid = 23; static constexpr int num_2d_vector_int = 13; #endif static constexpr int num_2d_vector_tr = 1; - uview_1d cell_length; uview_1d wpthlp_sfc; uview_1d wprtp_sfc; uview_1d upwp_sfc; @@ -509,6 +446,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess #ifdef SCREAM_SMALL_KERNELS uview_2d rho_zt; uview_2d shoc_qv; + uview_2d tabs; uview_2d dz_zt; uview_2d dz_zi; uview_2d tkh; @@ -524,6 +462,12 @@ class SHOCMacrophysics : public scream::AtmosphereProcess void initialize_impl (const RunType run_type); + // Update flux (if necessary) + void check_flux_state_consistency(const double dt); + + // Apply TMS drag coeff to shoc_main inputs (if necessary) + void apply_turbulent_mountain_stress (); + protected: void run_impl (const double dt); @@ -548,9 +492,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess Int m_num_tracers; Int hdtime; - KokkosTypes::view_1d m_cell_area; - KokkosTypes::view_1d m_cell_lat; - // Struct which contains local variables Buffer m_buffer; @@ -559,6 +500,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess SHF::SHOCInputOutput input_output; SHF::SHOCOutput output; SHF::SHOCHistoryOutput history_output; + SHF::SHOCRuntime runtime_options; #ifdef SCREAM_SMALL_KERNELS SHF::SHOCTemporaries temporaries; #endif diff --git a/components/eamxx/src/physics/shoc/eti/shoc_compute_shoc_temperature.cpp b/components/eamxx/src/physics/shoc/eti/shoc_compute_shoc_temperature.cpp new file mode 100644 index 000000000000..6b3e60f57dd7 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_compute_shoc_temperature.cpp @@ -0,0 +1,13 @@ +#include "shoc_compute_shoc_temperature_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for using the default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/impl/shoc_compute_diag_third_shoc_moment_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_compute_diag_third_shoc_moment_impl.hpp index 3d6a6062eb05..acd2922d05b1 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_compute_diag_third_shoc_moment_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_compute_diag_third_shoc_moment_impl.hpp @@ -13,6 +13,7 @@ ::compute_diag_third_shoc_moment( const MemberType& team, const Int& nlev, const Int& nlevi, + const Scalar& c_diag_3rd_mom, const uview_1d& w_sec, const uview_1d& thl_sec, const uview_1d& wthl_sec, @@ -44,7 +45,6 @@ ::compute_diag_third_shoc_moment( Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { // Constants - const auto c_diag_3rd_mom = scream::shoc::Constants::c_diag_3rd_mom; const Scalar a0 = (sp(0.52)*(1/(c_diag_3rd_mom*c_diag_3rd_mom)))/(c_diag_3rd_mom-2); const Scalar a1 = sp(0.87)/(c_diag_3rd_mom*c_diag_3rd_mom); const Scalar a2 = sp(0.5)/c_diag_3rd_mom; diff --git a/components/eamxx/src/physics/shoc/impl/shoc_compute_shoc_mix_shoc_length_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_compute_shoc_mix_shoc_length_impl.hpp index 43cd3ab9af0e..76ced9f901b8 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_compute_shoc_mix_shoc_length_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_compute_shoc_mix_shoc_length_impl.hpp @@ -12,6 +12,7 @@ void Functions ::compute_shoc_mix_shoc_length( const MemberType& team, const Int& nlev, + const Scalar& length_fac, const uview_1d& tke, const uview_1d& brunt, const uview_1d& zt_grid, @@ -20,7 +21,6 @@ ::compute_shoc_mix_shoc_length( { const Int nlev_pack = ekat::npack(nlev); const auto maxlen = scream::shoc::Constants::maxlen; - const auto length_fac = scream::shoc::Constants::length_fac; const auto vk = C::Karman; // Eddy turnover timescale diff --git a/components/eamxx/src/physics/shoc/impl/shoc_compute_shoc_temperature_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_compute_shoc_temperature_impl.hpp new file mode 100644 index 000000000000..c8f269a0662f --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_compute_shoc_temperature_impl.hpp @@ -0,0 +1,40 @@ +#ifndef SHOC_COMPUTE_SHOC_TEMPERATURE_IMPL_HPP +#define SHOC_COMPUTE_SHOC_TEMPERATURE_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc compute_shoc_temperature. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * This function computes absolute temperature + * based on SHOC's prognostic liquid water potential temperature. + */ + +template +KOKKOS_FUNCTION +void Functions::compute_shoc_temperature( + const MemberType& team, + const Int& nlev, + const uview_1d& thetal, + const uview_1d& ql, + const uview_1d& inv_exner, + const uview_1d& tabs) +{ + + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + + const Int nlev_pack = ekat::npack(nlev); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + tabs(k) = thetal(k)/inv_exner(k)+(lcond/cp)*ql(k); + }); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_diag_second_moments_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_diag_second_moments_impl.hpp index f4fa95744e4c..660f5dfc2331 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_diag_second_moments_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_diag_second_moments_impl.hpp @@ -15,6 +15,7 @@ template KOKKOS_FUNCTION void Functions::diag_second_moments( const MemberType& team, const Int& nlev, const Int& nlevi, + const Real& thl2tune, const Real& qw2tune, const Real& qwthl2tune, const Real& w2tune, const uview_1d& thetal, const uview_1d& qw, const uview_1d& u_wind, const uview_1d& v_wind, const uview_1d& tke, const uview_1d& isotropy, const uview_1d& tkh, const uview_1d& tk, const uview_1d& dz_zi, @@ -31,17 +32,6 @@ void Functions::diag_second_moments( // u, v, TKE, and tracers are computed here as well as the // correlation of qw and thetal. - const auto thl2tune = 1; - - // moisture variance - const auto qw2tune = 1; - - // temp moisture covariance - const auto qwthl2tune = 1; - - // vertical velocity variance - const auto w2tune = 1; - // Interpolate some variables from the midpoint grid to the interface grid linear_interp(team, zt_grid, zi_grid, isotropy, isotropy_zi, nlev, nlevi, 0); linear_interp(team, zt_grid, zi_grid, tkh, tkh_zi, nlev, nlevi, 0); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_diag_second_shoc_moments_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_diag_second_shoc_moments_impl.hpp index 2d35ea0de219..393564b238ea 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_diag_second_shoc_moments_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_diag_second_shoc_moments_impl.hpp @@ -14,6 +14,7 @@ namespace shoc { template KOKKOS_FUNCTION void Functions::diag_second_shoc_moments(const MemberType& team, const Int& nlev, const Int& nlevi, + const Scalar& thl2tune, const Scalar& qw2tune, const Scalar& qwthl2tune, const Scalar& w2tune, const uview_1d& thetal, const uview_1d& qw, const uview_1d& u_wind, const uview_1d& v_wind, const uview_1d& tke, const uview_1d& isotropy, const uview_1d& tkh, const uview_1d& tk, const uview_1d& dz_zi, @@ -58,6 +59,7 @@ void Functions::diag_second_shoc_moments(const MemberType& team, const Int& // Diagnose the second order moments, for points away from boundaries. this is // the main computation for the second moments diag_second_moments(team, nlev, nlevi, + thl2tune, qw2tune, qwthl2tune, w2tune, thetal, qw, u_wind,v_wind, tke, isotropy,tkh, tk, dz_zi, zt_grid, zi_grid, shoc_mix, isotropy_zi, tkh_zi, tk_zi, thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_diag_third_shoc_moments_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_diag_third_shoc_moments_impl.hpp index a32982a5e9dd..ecdee4f64661 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_diag_third_shoc_moments_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_diag_third_shoc_moments_impl.hpp @@ -17,6 +17,7 @@ void Functions::diag_third_shoc_moments( const MemberType& team, const Int& nlev, const Int& nlevi, + const Scalar& c_diag_3rd_mom, const uview_1d& w_sec, const uview_1d& thl_sec, const uview_1d& wthl_sec, @@ -49,7 +50,7 @@ void Functions::diag_third_shoc_moments( team.team_barrier(); // Diagnose the third moment of the vertical-velocity - compute_diag_third_shoc_moment(team,nlev,nlevi,w_sec,thl_sec,wthl_sec, + compute_diag_third_shoc_moment(team,nlev,nlevi,c_diag_3rd_mom,w_sec,thl_sec,wthl_sec, tke, dz_zt, dz_zi,isotropy_zi, brunt_zi, w_sec_zi,thetal_zi,w3); team.team_barrier(); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_eddy_diffusivities_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_eddy_diffusivities_impl.hpp index fea50b23dd16..80244c22c234 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_eddy_diffusivities_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_eddy_diffusivities_impl.hpp @@ -16,9 +16,11 @@ KOKKOS_FUNCTION void Functions::eddy_diffusivities( const MemberType& team, const Int& nlev, - const Scalar& obklen, + const Scalar& Ckh, + const Scalar& Ckm, const Scalar& pblh, const uview_1d& zt_grid, + const uview_1d& tabs, const uview_1d& shoc_mix, const uview_1d& sterm_zt, const uview_1d& isotropy, @@ -28,44 +30,23 @@ void Functions::eddy_diffusivities( { // Parameters - // Critical value of dimensionless Monin-Obukhov length, - // for which diffusivities are no longer damped - const Int zL_crit_val = 100; + // Minimum absolute temperature [K] to which apply extra mixing + const Int tabs_crit = 182; // Transition depth [m] above PBL top to allow // stability diffusivities const Int pbl_trans = 200; - // Turbulent coefficients - const Scalar Ckh = 0.1; - const Scalar Ckm = 0.1; - // Maximum eddy coefficients for stable PBL diffusivities - const Scalar Ckh_s_max = 0.1; - const Scalar Ckm_s_max = 0.1; - // Minimum allowable value for stability diffusivities - const Scalar Ckh_s_min = 0.1; - const Scalar Ckm_s_min = 0.1; + // Dddy coefficients for stable PBL diffusivities + const Scalar Ckh_s = 0.1; + const Scalar Ckm_s = 0.1; - const auto s_zt_grid = ekat::scalarize(zt_grid); + const auto s_tabs = ekat::scalarize(tabs); const Int nlev_pack = ekat::npack(nlev); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { - // Dimensionless Okukhov length considering only - // the lowest model grid layer height to scale - const auto z_over_L = s_zt_grid(nlev-1)/obklen; - - // Compute diffusivity coefficient as function of dimensionless Obukhov, - // given a critical value - const Scalar Ckh_s = ekat::impl::max(Ckh_s_min, - ekat::impl::min(Ckh_s_max, - z_over_L/zL_crit_val)); - const Scalar Ckm_s = ekat::impl::max(Ckm_s_min, - ekat::impl::min(Ckm_s_max, - z_over_L/zL_crit_val)); - - // If surface layer is stable, based on near surface dimensionless Monin-Obukov - // use modified coefficients of tkh and tk that are primarily based on shear - // production and SHOC length scale, to promote mixing within the PBL and to a - // height slighty above to ensure smooth transition. - const Smask condition = (zt_grid(k) < pblh+pbl_trans) && (z_over_L > 0); + // If surface layer temperature is running away, apply extra mixing + // based on traditional stable PBL diffusivities that are not damped + // by stability functions. + const Smask condition = (zt_grid(k) < pblh+pbl_trans) && (s_tabs(nlev-1) < tabs_crit); tkh(k).set(condition, Ckh_s*ekat::square(shoc_mix(k))*ekat::sqrt(sterm_zt(k))); tk(k).set(condition, Ckm_s*ekat::square(shoc_mix(k))*ekat::sqrt(sterm_zt(k))); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_energy_fixer_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_energy_fixer_impl.hpp index eb2f85112221..324c672d2e35 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_energy_fixer_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_energy_fixer_impl.hpp @@ -2,8 +2,12 @@ #define SHOC_ENERGY_FIXER_IMPL_HPP #include "shoc_functions.hpp" // for ETI only but harmless for GPU +#include "share/util/scream_common_physics_functions.hpp" namespace scream { + +using PF = scream::PhysicsFunctions; + namespace shoc { /* @@ -65,8 +69,10 @@ void Functions::shoc_energy_fixer( const auto s_rho_zi = ekat::scalarize(rho_zi); const auto s_pint = ekat::scalarize(pint); + const auto exner_int = PF::exner_function(s_pint(nlevi-1)); + // Compute the total energy before and after SHOC call - const Scalar shf = wthl_sfc*cp*s_rho_zi(nlevi-1); + const Scalar shf = wthl_sfc*cp*s_rho_zi(nlevi-1)*exner_int; const Scalar lhf = wqw_sfc*s_rho_zi(nlevi-1); te_a = se_a + ke_a + (lcond+lice)*wv_a + lice*wl_a; te_b = se_b + ke_b + (lcond+lice)*wv_b + lice*wl_b; diff --git a/components/eamxx/src/physics/shoc/impl/shoc_isotropic_ts_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_isotropic_ts_impl.hpp index 948253de776c..1ba9ca49070c 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_isotropic_ts_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_isotropic_ts_impl.hpp @@ -17,6 +17,10 @@ void Functions ::isotropic_ts( const MemberType& team, const Int& nlev, + const Scalar& lambda_low_in, + const Scalar& lambda_high_in, + const Scalar& lambda_slope_in, + const Scalar& lambda_thresh_in, const Scalar& brunt_int, const uview_1d& tke, const uview_1d& a_diss, @@ -28,11 +32,11 @@ ::isotropic_ts( static constexpr Scalar ggr = C::gravit; //Declare constants - static constexpr Scalar lambda_low = 0.001; - static constexpr Scalar lambda_high = 0.04; - static constexpr Scalar lambda_slope = 2.65; - static constexpr Scalar lambda_thresh= 0.02; - static constexpr Scalar maxiso = 20000; // Return to isotropic timescale [s] + const Scalar lambda_low = lambda_low_in; + const Scalar lambda_high = lambda_high_in; + const Scalar lambda_slope = lambda_slope_in; + const Scalar lambda_thresh = lambda_thresh_in; + static constexpr Scalar maxiso = 20000; // Return to isotropic timescale [s] const Int nlev_pack = ekat::npack(nlev); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_length_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_length_impl.hpp index 86288b9eecc3..e52554383fe7 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_length_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_length_impl.hpp @@ -13,6 +13,7 @@ ::shoc_length( const MemberType& team, const Int& nlev, const Int& nlevi, + const Scalar& length_fac, const Scalar& dx, const Scalar& dy, const uview_1d& zt_grid, @@ -36,7 +37,7 @@ ::shoc_length( Scalar l_inf = 0; compute_l_inf_shoc_length(team,nlev,zt_grid,dz_zt,tke,l_inf); - compute_shoc_mix_shoc_length(team,nlev,tke,brunt,zt_grid,l_inf,shoc_mix); + compute_shoc_mix_shoc_length(team,nlev,length_fac, tke,brunt,zt_grid,l_inf,shoc_mix); team.team_barrier(); check_length_scale_shoc_length(team,nlev,dx,dy,shoc_mix); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 430b427265cf..13a93992cff1 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -72,6 +72,19 @@ void Functions::shoc_main_internal( const Int& nadv, // Number of times to loop SHOC const Int& num_qtracers, // Number of tracers const Scalar& dtime, // SHOC timestep [s] + // Runtime Parameters + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, + const Scalar& thl2tune, + const Scalar& qw2tune, + const Scalar& qwthl2tune, + const Scalar& w2tune, + const Scalar& length_fac, + const Scalar& c_diag_3rd_mom, + const Scalar& Ckh, + const Scalar& Ckm, // Input Variables const Scalar& dx, const Scalar& dy, @@ -124,10 +137,10 @@ void Functions::shoc_main_internal( { // Define temporary variables - uview_1d rho_zt, shoc_qv, dz_zt, dz_zi, tkh; - workspace.template take_many_and_reset<5>( - {"rho_zt", "shoc_qv", "dz_zt", "dz_zi", "tkh"}, - {&rho_zt, &shoc_qv, &dz_zt, &dz_zi, &tkh}); + uview_1d rho_zt, shoc_qv, shoc_tabs, dz_zt, dz_zi, tkh; + workspace.template take_many_and_reset<6>( + {"rho_zt", "shoc_qv", "shoc_tabs", "dz_zt", "dz_zi", "tkh"}, + {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi, &tkh}); // Local scalars Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, @@ -167,6 +180,11 @@ void Functions::shoc_main_internal( compute_shoc_vapor(team,nlev,qw,shoc_ql, // Input shoc_qv); // Output + // Update SHOC temperature + compute_shoc_temperature(team,nlev,thetal, // Input + shoc_ql,inv_exner, // Input + shoc_tabs); // Output + team.team_barrier(); shoc_diag_obklen(uw_sfc,vw_sfc, // Input wthl_sfc, wqw_sfc, // Input @@ -184,20 +202,25 @@ void Functions::shoc_main_internal( pblh); // Output // Update the turbulent length scale - shoc_length(team,nlev,nlevi,dx,dy, // Input + shoc_length(team,nlev,nlevi, // Input + length_fac, // Runtime Options + dx,dy, // Input zt_grid,zi_grid,dz_zt, // Input tke,thv, // Input workspace, // Workspace brunt,shoc_mix); // Output // Advance the SGS TKE equation - shoc_tke(team,nlev,nlevi,dtime,wthv_sec, // Input - shoc_mix,dz_zi,dz_zt,pres,u_wind, // Input - v_wind,brunt,obklen,zt_grid, // Input - zi_grid,pblh, // Input - workspace, // Workspace - tke,tk,tkh, // Input/Output - isotropy); // Output + shoc_tke(team,nlev,nlevi,dtime, // Input + lambda_low,lambda_high,lambda_slope, // Runtime options + lambda_thresh,Ckh,Ckm, // Runtime options + wthv_sec, // Input + shoc_mix,dz_zi,dz_zt,pres,shoc_tabs,// Input + u_wind,v_wind,brunt,zt_grid, // Input + zi_grid,pblh, // Input + workspace, // Workspace + tke,tk,tkh, // Input/Output + isotropy); // Output // Update SHOC prognostic variables here // via implicit diffusion solver @@ -209,7 +232,9 @@ void Functions::shoc_main_internal( thetal,qw,qtracers,tke,u_wind,v_wind); // Input/Output // Diagnose the second order moments - diag_second_shoc_moments(team,nlev,nlevi,thetal,qw,u_wind,v_wind, // Input + diag_second_shoc_moments(team,nlev,nlevi, + thl2tune, qw2tune, qwthl2tune, w2tune, // Runtime options + thetal,qw,u_wind,v_wind, // Input tke,isotropy,tkh,tk,dz_zi,zt_grid,zi_grid, // Input shoc_mix,wthl_sfc,wqw_sfc,uw_sfc,vw_sfc, // Input ustar2,wstar, // Input/Output @@ -219,7 +244,9 @@ void Functions::shoc_main_internal( // Diagnose the third moment of vertical velocity, // needed for the PDF closure - diag_third_shoc_moments(team,nlev,nlevi,w_sec,thl_sec,wthl_sec, // Input + diag_third_shoc_moments(team,nlev,nlevi, + c_diag_3rd_mom, // Runtime options + w_sec,thl_sec,wthl_sec, // Input isotropy,brunt,thetal,tke,dz_zt,dz_zi, // Input zt_grid,zi_grid, // Input workspace, // Workspace @@ -284,8 +311,8 @@ void Functions::shoc_main_internal( pblh); // Output // Release temporary variables from the workspace - workspace.template release_many_contiguous<5>( - {&rho_zt, &shoc_qv, &dz_zt, &dz_zi, &tkh}); + workspace.template release_many_contiguous<6>( + {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi, &tkh}); } #else template @@ -297,6 +324,19 @@ void Functions::shoc_main_internal( const Int& nadv, // Number of times to loop SHOC const Int& num_qtracers, // Number of tracers const Scalar& dtime, // SHOC timestep [s] + // Runtime Parameters + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, + const Scalar& thl2tune, + const Scalar& qw2tune, + const Scalar& qwthl2tune, + const Scalar& w2tune, + const Scalar& length_fac, + const Scalar& c_diag_3rd_mom, + const Scalar& Ckh, + const Scalar& Ckm, // Input Variables const view_1d& dx, const view_1d& dy, @@ -362,6 +402,7 @@ void Functions::shoc_main_internal( const view_1d& wstar, const view_2d& rho_zt, const view_2d& shoc_qv, + const view_2d& shoc_tabs, const view_2d& dz_zt, const view_2d& dz_zi, const view_2d& tkh) @@ -399,6 +440,11 @@ void Functions::shoc_main_internal( compute_shoc_vapor_disp(shcol,nlev,qw,shoc_ql, // Input shoc_qv); // Output + // Update SHOC temperature + compute_shoc_temperature_disp(shcol,nlev,thetal, // Input + shoc_ql,inv_exner, // Input + shoc_tabs); // Output + shoc_diag_obklen_disp(shcol, nlev, uw_sfc,vw_sfc, // Input wthl_sfc, wqw_sfc, // Input @@ -416,20 +462,25 @@ void Functions::shoc_main_internal( pblh); // Output // Update the turbulent length scale - shoc_length_disp(shcol,nlev,nlevi,dx,dy, // Input + shoc_length_disp(shcol,nlev,nlevi, // Input + length_fac, // Runtime Options + dx,dy, // Input zt_grid,zi_grid,dz_zt, // Input tke,thv, // Input workspace_mgr, // Workspace mgr brunt,shoc_mix); // Output // Advance the SGS TKE equation - shoc_tke_disp(shcol,nlev,nlevi,dtime,wthv_sec, // Input - shoc_mix,dz_zi,dz_zt,pres,u_wind, // Input - v_wind,brunt,obklen,zt_grid, // Input - zi_grid,pblh, // Input - workspace_mgr, // Workspace mgr - tke,tk,tkh, // Input/Output - isotropy); // Output + shoc_tke_disp(shcol,nlev,nlevi,dtime, // Input + lambda_low,lambda_high,lambda_slope, // Runtime options + lambda_thresh,Ckh,Ckm, // Runtime options + wthv_sec, // Input + shoc_mix,dz_zi,dz_zt,pres,shoc_tabs,// Input + u_wind,v_wind,brunt,zt_grid, // Input + zi_grid,pblh, // Input + workspace_mgr, // Workspace mgr + tke,tk,tkh, // Input/Output + isotropy); // Output // Update SHOC prognostic variables here // via implicit diffusion solver @@ -440,7 +491,9 @@ void Functions::shoc_main_internal( thetal,qw,qtracers,tke,u_wind,v_wind); // Input/Output // Diagnose the second order moments - diag_second_shoc_moments_disp(shcol,nlev,nlevi,thetal,qw,u_wind,v_wind, // Input + diag_second_shoc_moments_disp(shcol,nlev,nlevi, + thl2tune, qw2tune, qwthl2tune, w2tune, // Runtime options + thetal,qw,u_wind,v_wind, // Input tke,isotropy,tkh,tk,dz_zi,zt_grid,zi_grid, // Input shoc_mix,wthl_sfc,wqw_sfc,uw_sfc,vw_sfc, // Input ustar2,wstar, // Input/Output @@ -450,7 +503,9 @@ void Functions::shoc_main_internal( // Diagnose the third moment of vertical velocity, // needed for the PDF closure - diag_third_shoc_moments_disp(shcol,nlev,nlevi,w_sec,thl_sec,wthl_sec, // Input + diag_third_shoc_moments_disp(shcol,nlev,nlevi, + c_diag_3rd_mom, // Runtime options + w_sec,thl_sec,wthl_sec, // Input isotropy,brunt,thetal,tke,dz_zt,dz_zi, // Input zt_grid,zi_grid, // Input workspace_mgr, // Workspace mgr @@ -523,6 +578,7 @@ Int Functions::shoc_main( const Int& num_qtracers, // Number of tracers const Scalar& dtime, // SHOC timestep [s] WorkspaceMgr& workspace_mgr, // WorkspaceManager for local variables + const SHOCRuntime& shoc_runtime, // Runtime Options const SHOCInput& shoc_input, // Input const SHOCInputOutput& shoc_input_output, // Input/Output const SHOCOutput& shoc_output, // Output @@ -535,6 +591,20 @@ Int Functions::shoc_main( // Start timer auto start = std::chrono::steady_clock::now(); + // Runtime options + const Scalar lambda_low = shoc_runtime.lambda_low; + const Scalar lambda_high = shoc_runtime.lambda_high; + const Scalar lambda_slope = shoc_runtime.lambda_slope; + const Scalar lambda_thresh = shoc_runtime.lambda_thresh; + const Scalar thl2tune = shoc_runtime.thl2tune; + const Scalar qw2tune = shoc_runtime.qw2tune; + const Scalar qwthl2tune = shoc_runtime.qwthl2tune; + const Scalar w2tune = shoc_runtime.w2tune; + const Scalar length_fac = shoc_runtime.length_fac; + const Scalar c_diag_3rd_mom = shoc_runtime.c_diag_3rd_mom; + const Scalar Ckh = shoc_runtime.Ckh; + const Scalar Ckm = shoc_runtime.Ckm; + #ifndef SCREAM_SMALL_KERNELS using ExeSpace = typename KT::ExeSpace; @@ -593,6 +663,9 @@ Int Functions::shoc_main( const auto qtracers_s = Kokkos::subview(shoc_input_output.qtracers, i, Kokkos::ALL(), Kokkos::ALL()); shoc_main_internal(team, nlev, nlevi, npbl, nadv, num_qtracers, dtime, + lambda_low, lambda_high, lambda_slope, lambda_thresh, // Runtime options + thl2tune, qw2tune, qwthl2tune, w2tune, length_fac, // Runtime options + c_diag_3rd_mom, Ckh, Ckm, // Runtime options dx_s, dy_s, zt_grid_s, zi_grid_s, // Input pres_s, presi_s, pdel_s, thv_s, w_field_s, // Input wthl_sfc_s, wqw_sfc_s, uw_sfc_s, vw_sfc_s, // Input @@ -614,6 +687,9 @@ Int Functions::shoc_main( const auto v_wind_s = Kokkos::subview(shoc_input_output.horiz_wind, Kokkos::ALL(), 1, Kokkos::ALL()); shoc_main_internal(shcol, nlev, nlevi, npbl, nadv, num_qtracers, dtime, + lambda_low, lambda_high, lambda_slope, lambda_thresh, // Runtime options + thl2tune, qw2tune, qwthl2tune, w2tune, length_fac, // Runtime options + c_diag_3rd_mom, Ckh, Ckm, // Runtime options shoc_input.dx, shoc_input.dy, shoc_input.zt_grid, shoc_input.zi_grid, // Input shoc_input.pres, shoc_input.presi, shoc_input.pdel, shoc_input.thv, shoc_input.w_field, // Input shoc_input.wthl_sfc, shoc_input.wqw_sfc, shoc_input.uw_sfc, shoc_input.vw_sfc, // Input @@ -630,8 +706,8 @@ Int Functions::shoc_main( shoc_temporaries.se_b, shoc_temporaries.ke_b, shoc_temporaries.wv_b, shoc_temporaries.wl_b, shoc_temporaries.se_a, shoc_temporaries.ke_a, shoc_temporaries.wv_a, shoc_temporaries.wl_a, shoc_temporaries.ustar, shoc_temporaries.kbfs, shoc_temporaries.obklen, shoc_temporaries.ustar2, - shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, shoc_temporaries.dz_zt, - shoc_temporaries.dz_zi, shoc_temporaries.tkh); + shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, + shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi, shoc_temporaries.tkh); #endif auto finish = std::chrono::steady_clock::now(); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_tke_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_tke_impl.hpp index 75910581d349..1b87a08efa15 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_tke_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_tke_impl.hpp @@ -24,15 +24,21 @@ void Functions::shoc_tke( const Int& nlev, const Int& nlevi, const Scalar& dtime, + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, + const Scalar& Ckh, + const Scalar& Ckm, const uview_1d& wthv_sec, const uview_1d& shoc_mix, const uview_1d& dz_zi, const uview_1d& dz_zt, const uview_1d& pres, + const uview_1d& tabs, const uview_1d& u_wind, const uview_1d& v_wind, const uview_1d& brunt, - const Scalar& obklen, const uview_1d& zt_grid, const uview_1d& zi_grid, const Scalar& pblh, @@ -64,10 +70,10 @@ void Functions::shoc_tke( adv_sgs_tke(team,nlev,dtime,shoc_mix,wthv_sec,sterm_zt,tk,tke,a_diss); // Compute isotropic time scale [s] - isotropic_ts(team,nlev,brunt_int,tke,a_diss,brunt,isotropy); + isotropic_ts(team,nlev,lambda_low,lambda_high,lambda_slope,lambda_thresh,brunt_int,tke,a_diss,brunt,isotropy); // Compute eddy diffusivity for heat and momentum - eddy_diffusivities(team,nlev,obklen,pblh,zt_grid,shoc_mix,sterm_zt,isotropy,tke,tkh,tk); + eddy_diffusivities(team,nlev,Ckh,Ckm,pblh,zt_grid,tabs,shoc_mix,sterm_zt,isotropy,tke,tkh,tk); // Release temporary variables from the workspace workspace.template release_many_contiguous<3>( diff --git a/components/eamxx/src/physics/shoc/shoc_constants.hpp b/components/eamxx/src/physics/shoc/shoc_constants.hpp index 5f934a508520..d8f0245e9625 100644 --- a/components/eamxx/src/physics/shoc/shoc_constants.hpp +++ b/components/eamxx/src/physics/shoc/shoc_constants.hpp @@ -16,8 +16,6 @@ struct Constants static constexpr Scalar minlen = 20.0; // Lower limit for mixing length [m] static constexpr Scalar maxlen = 20000.0; // Upper limit for mixing length [m] static constexpr Scalar maxiso = 20000.0; // Upper limit for isotropy time scale [s] - static constexpr Scalar length_fac = 0.5; // Mixing length scaling parameter - static constexpr Scalar c_diag_3rd_mom = 7.0; // Coefficient for diag third moment parameters static constexpr Scalar w3clip = 1.2; // Third moment of vertical velocity static constexpr Scalar ustar_min = 0.01; // Minimum surface friction velocity static constexpr Scalar largeneg = -99999999.99; // Large negative value used for linear_interp threshold diff --git a/components/eamxx/src/physics/shoc/shoc_f90.cpp b/components/eamxx/src/physics/shoc/shoc_f90.cpp index bef535c8ac71..bc14a9110d52 100644 --- a/components/eamxx/src/physics/shoc/shoc_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_f90.cpp @@ -8,7 +8,7 @@ using scream::Real; using scream::Int; extern "C" { void shoc_init_c(int nlev, Real gravit, Real rair, Real rh2o, Real cpair, - Real zvir, Real latvap, Real latice, Real karman); + Real zvir, Real latvap, Real latice, Real karman, Real p0); void shoc_use_cxx_c(bool use_cxx); } @@ -119,7 +119,7 @@ void shoc_init(Int nlev, bool use_fortran, bool force_reinit) { using C = scream::physics::Constants; shoc_init_c((int)nlev, C::gravit, C::Rair, C::RH2O, C::Cpair, C::ZVIR, - C::LatVap, C::LatIce, C::Karman); + C::LatVap, C::LatIce, C::Karman, C::P0); is_init = true; } shoc_use_cxx_c(!use_fortran); diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index e5162e83fcb1..d02d498c5a43 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -69,6 +69,24 @@ struct Functions using WorkspaceMgr = typename ekat::WorkspaceManager; using Workspace = typename WorkspaceMgr::Workspace; + // This struct stores runtime options for shoc_main + struct SHOCRuntime { + SHOCRuntime() = default; + // Runtime options for isotropic_ts + Scalar lambda_low; + Scalar lambda_high; + Scalar lambda_slope; + Scalar lambda_thresh; + Scalar thl2tune; + Scalar qw2tune; + Scalar qwthl2tune; + Scalar w2tune; + Scalar length_fac; + Scalar c_diag_3rd_mom; + Scalar Ckh; + Scalar Ckm; + }; + // This struct stores input views for shoc_main. struct SHOCInput { SHOCInput() = default; @@ -199,6 +217,7 @@ struct Functions view_2d rho_zt; view_2d shoc_qv; + view_2d tabs; view_2d dz_zt; view_2d dz_zi; view_2d tkh; @@ -266,6 +285,7 @@ struct Functions const MemberType& team, const Int& nlev, const Int& nlevi, + const Scalar& c_diag_3rd_mom, const uview_1d& w_sec, const uview_1d& thl_sec, const uview_1d& wthl_sec, @@ -288,6 +308,7 @@ struct Functions static void compute_shoc_mix_shoc_length( const MemberType& team, const Int& nlev, + const Scalar& length_fac, const uview_1d& tke, const uview_1d& brunt, const uview_1d& zt_grid, @@ -363,6 +384,7 @@ struct Functions KOKKOS_FUNCTION static void diag_second_moments(const MemberType& team, const Int& nlev, const Int& nlevi, + const Real& thl2tune, const Real& qw2tune, const Real& qwthl2tune, const Real& w2tune, const uview_1d& thetal, const uview_1d& qw, const uview_1d& u_wind, const uview_1d& v_wind, const uview_1d& tke, const uview_1d& isotropy, const uview_1d& tkh, const uview_1d& tk, const uview_1d& dz_zi, @@ -374,6 +396,7 @@ struct Functions KOKKOS_FUNCTION static void diag_second_shoc_moments(const MemberType& team, const Int& nlev, const Int& nlevi, + const Scalar& thl2tune, const Scalar& qw2tune, const Scalar& qwthl2tune, const Scalar& w2tune, const uview_1d& thetal, const uview_1d& qw, const uview_1d& u_wind, const uview_1d& v_wind, const uview_1d& tke, const uview_1d& isotropy, const uview_1d& tkh, const uview_1d& tk, const uview_1d& dz_zi, @@ -385,6 +408,10 @@ struct Functions #ifdef SCREAM_SMALL_KERNELS static void diag_second_shoc_moments_disp( const Int& shcol, const Int& nlev, const Int& nlevi, + const Scalar& thl2tune, + const Scalar& qw2tune, + const Scalar& qwthl2tune, + const Scalar& w2tune, const view_2d& thetal, const view_2d& qw, const view_2d& u_wind, @@ -480,6 +507,7 @@ struct Functions const MemberType& team, const Int& nlev, const Int& nlevi, + const Scalar& length_fac, const Scalar& dx, const Scalar& dy, const uview_1d& zt_grid, @@ -495,6 +523,7 @@ struct Functions const Int& shcol, const Int& nlev, const Int& nlevi, + const Scalar& length_fac, const view_1d& dx, const view_1d& dy, const view_2d& zt_grid, @@ -573,6 +602,24 @@ struct Functions const view_2d& qv); #endif + KOKKOS_FUNCTION + static void compute_shoc_temperature( + const MemberType& team, + const Int& nlev, + const uview_1d& thetal, + const uview_1d& ql, + const uview_1d& inv_exner, + const uview_1d& tabs); +#ifdef SCREAM_SMALL_KERNELS + static void compute_shoc_temperature_disp( + const Int& shcol, + const Int& nlev, + const view_2d& thetal, + const view_2d& ql, + const view_2d& inv_exner, + const view_2d& tabs); +#endif + KOKKOS_FUNCTION static void update_prognostics_implicit( const MemberType& team, @@ -632,6 +679,7 @@ struct Functions const MemberType& team, const Int& nlev, const Int& nlevi, + const Scalar& c_diag_3rd_mom, const uview_1d& w_sec, const uview_1d& thl_sec, const uview_1d& wthl_sec, @@ -650,6 +698,7 @@ struct Functions const Int& shcol, const Int& nlev, const Int& nlevi, + const Scalar& c_diag_3rd_mom, const view_2d& w_sec, const view_2d& thl_sec, const view_2d& wthl_sec, @@ -759,6 +808,10 @@ struct Functions static void isotropic_ts( const MemberType& team, const Int& nlev, + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, const Scalar& brunt_int, const uview_1d& tke, const uview_1d& a_diss, @@ -788,6 +841,19 @@ struct Functions const Int& nadv, // Number of times to loop SHOC const Int& num_qtracers, // Number of tracers const Scalar& dtime, // SHOC timestep [s] + // Runtime Parameters + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, + const Scalar& thl2tune, + const Scalar& qw2tune, + const Scalar& qwthl2tune, + const Scalar& w2tune, + const Scalar& length_fac, + const Scalar& c_diag_3rd_mom, + const Scalar& Ckh, + const Scalar& Ckm, // Input Variables const Scalar& host_dx, const Scalar& host_dy, @@ -846,6 +912,19 @@ struct Functions const Int& nadv, // Number of times to loop SHOC const Int& num_qtracers, // Number of tracers const Scalar& dtime, // SHOC timestep [s] + // Runtime Parameters + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, + const Scalar& thl2tune, + const Scalar& qw2tune, + const Scalar& qwthl2tune, + const Scalar& w2tune, + const Scalar& length_fac, + const Scalar& c_diag_3rd_mom, + const Scalar& Ckh, + const Scalar& Ckm, // Input Variables const view_1d& host_dx, const view_1d& host_dy, @@ -911,6 +990,7 @@ struct Functions const view_1d& wstar, const view_2d& rho_zt, const view_2d& shoc_qv, + const view_2d& tabs, const view_2d& dz_zt, const view_2d& dz_zi, const view_2d& tkh); @@ -926,6 +1006,7 @@ struct Functions const Int& num_q_tracers, // Number of tracers const Scalar& dtime, // SHOC timestep [s] WorkspaceMgr& workspace_mgr, // WorkspaceManager for local variables + const SHOCRuntime& shoc_runtime, // Runtime options const SHOCInput& shoc_input, // Input const SHOCInputOutput& shoc_input_output, // Input/Output const SHOCOutput& shoc_output, // Output @@ -1050,9 +1131,11 @@ struct Functions static void eddy_diffusivities( const MemberType& team, const Int& nlev, - const Scalar& obklen, + const Scalar& Ckh, + const Scalar& Ckm, const Scalar& pblh, const uview_1d& zt_grid, + const uview_1d& tabs, const uview_1d& shoc_mix, const uview_1d& sterm_zt, const uview_1d& isotropy, @@ -1066,15 +1149,21 @@ struct Functions const Int& nlev, const Int& nlevi, const Scalar& dtime, + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, + const Scalar& Ckh, + const Scalar& Ckm, const uview_1d& wthv_sec, const uview_1d& shoc_mix, const uview_1d& dz_zi, const uview_1d& dz_zt, const uview_1d& pres, + const uview_1d& tabs, const uview_1d& u_wind, const uview_1d& v_wind, const uview_1d& brunt, - const Scalar& obklen, const uview_1d& zt_grid, const uview_1d& zi_grid, const Scalar& pblh, @@ -1089,15 +1178,21 @@ struct Functions const Int& nlev, const Int& nlevi, const Scalar& dtime, + const Scalar& lambda_low, + const Scalar& lambda_high, + const Scalar& lambda_slope, + const Scalar& lambda_thresh, + const Scalar& Ckh, + const Scalar& Ckm, const view_2d& wthv_sec, const view_2d& shoc_mix, const view_2d& dz_zi, const view_2d& dz_zt, const view_2d& pres, + const view_2d& tabs, const view_2d& u_wind, const view_2d& v_wind, const view_2d& brunt, - const view_1d& obklen, const view_2d& zt_grid, const view_2d& zi_grid, const view_1d& pblh, @@ -1115,7 +1210,7 @@ struct Functions // If a GPU build, without relocatable device code enabled, make all code available // to the translation unit; otherwise, ETI is used. #if defined(EAMXX_ENABLE_GPU) && !defined(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) \ - && !defined(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) + && !defined(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) # include "shoc_calc_shoc_varorcovar_impl.hpp" # include "shoc_calc_shoc_vertflux_impl.hpp" @@ -1158,6 +1253,8 @@ struct Functions # include "shoc_grid_impl.hpp" # include "shoc_eddy_diffusivities_impl.hpp" # include "shoc_tke_impl.hpp" -#endif // GPU || !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE +# include "shoc_compute_shoc_temperature_impl.hpp" + +#endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE #endif // SHOC_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index e17a0d20c4f1..0e4acceb1a79 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -22,7 +22,7 @@ extern "C" { // Special shoc_init function for shoc_main_bfb test void shoc_init_for_main_bfb_c(int nlev, Real gravit, Real rair, Real rh2o, Real cpair, - Real zvir, Real latvap, Real latice, Real karman, + Real zvir, Real latvap, Real latice, Real karman, Real p0, Real* pref_mid, int nbot_shoc, int ntop_shoc); void shoc_use_cxx_c(bool use_cxx); @@ -52,7 +52,7 @@ void shoc_energy_total_fixer_c(Int shcol, Int nlev, Int nlevi, Real dtime, Int n Real *zt_grid, Real *zi_grid, Real *se_b, Real *ke_b, Real *wv_b, Real *wl_b, Real *se_a, Real *ke_a, Real *wv_a, Real *wl_a, - Real *wthl_sfc, Real *wqw_sfc, Real *rho_zt, + Real *wthl_sfc, Real *wqw_sfc, Real *rho_zt, Real *pint, Real *te_a, Real *te_b); void shoc_energy_threshold_fixer_c(Int shcol, Int nlev, Int nlevi, @@ -87,7 +87,7 @@ void check_tke_c(Int shcol, Int nlev, Real *tke); void shoc_tke_c(Int shcol, Int nlev, Int nlevi, Real dtime, Real *wthv_sec, Real *shoc_mix, Real *dz_zi, Real *dz_zt, Real *pres, - Real *u_wind, Real *v_wind, Real *brunt, Real *obklen, + Real* tabs, Real *u_wind, Real *v_wind, Real *brunt, Real *zt_grid, Real *zi_grid, Real *pblh, Real *tke, Real *tk, Real *tkh, Real *isotropy); @@ -104,8 +104,8 @@ void adv_sgs_tke_c(Int nlev, Int shcol, Real dtime, Real *shoc_mix, Real *wthv_sec, Real *sterm_zt, Real *tk, Real *tke, Real *a_diss); -void eddy_diffusivities_c(Int nlev, Int shcol, Real *obklen, Real *pblh, - Real *zt_grid, Real *shoc_mix, Real *sterm_zt, +void eddy_diffusivities_c(Int nlev, Int shcol, Real *pblh, + Real *zt_grid, Real *tabs, Real *shoc_mix, Real *sterm_zt, Real *isotropy, Real *tke, Real *tkh, Real *tk); void calc_shoc_vertflux_c(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, @@ -283,6 +283,9 @@ void vd_shoc_solve_c(Int shcol, Int nlev, Real* du, Real* dl, Real* d, Real* var void pblintd_surf_temp_c(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino); void pblintd_check_pblh_c(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, bool* check, Real* pblh); void pblintd_c(Int shcol, Int nlev, Int nlevi, Int npbl_in, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh); + +void compute_shoc_temperature_c(Int shcol, Int nlev, Real* thetal, Real*ql, Real* inv_exner, Real* tabs); + } // extern "C" : end _c decls namespace scream { @@ -344,7 +347,10 @@ void shoc_energy_total_fixer(ShocEnergyTotalFixerData& d) { shoc_init(d.nlev, true); d.transpose(); - shoc_energy_total_fixer_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, d.wqw_sfc, d.rho_zt, d.te_a, d.te_b); + shoc_energy_total_fixer_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, + d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, + d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, + d.wqw_sfc, d.rho_zt, d.pint, d.te_a, d.te_b); d.transpose(); } @@ -436,7 +442,7 @@ void shoc_tke(ShocTkeData& d) { shoc_init(d.nlev, true); d.transpose(); - shoc_tke_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, d.pres, d.u_wind, d.v_wind, d.brunt, d.obklen, d.zt_grid, d.zi_grid, d.pblh, d.tke, d.tk, d.tkh, d.isotropy); + shoc_tke_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, d.pres, d.tabs, d.u_wind, d.v_wind, d.brunt, d.zt_grid, d.zi_grid, d.pblh, d.tke, d.tk, d.tkh, d.isotropy); d.transpose(); } @@ -468,7 +474,7 @@ void eddy_diffusivities(EddyDiffusivitiesData& d) { shoc_init(d.nlev, true); d.transpose(); - eddy_diffusivities_c(d.nlev, d.shcol, d.obklen, d.pblh, d.zt_grid, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); + eddy_diffusivities_c(d.nlev, d.shcol, d.pblh, d.zt_grid, d.tabs, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); d.transpose(); } @@ -751,7 +757,7 @@ void shoc_main_with_init(ShocMainData& d) using C = scream::physics::Constants; d.transpose(); - shoc_init_for_main_bfb_c(d.nlev, C::gravit, C::Rair, C::RH2O, C::Cpair, C::ZVIR, C::LatVap, C::LatIce, C::Karman, + shoc_init_for_main_bfb_c(d.nlev, C::gravit, C::Rair, C::RH2O, C::Cpair, C::ZVIR, C::LatVap, C::LatIce, C::Karman, C::P0, d.pref_mid, d.nbot_shoc, d.ntop_shoc+1); shoc_use_cxx_c(false); @@ -821,6 +827,14 @@ void pblintd(PblintdData& d) d.transpose(); } +void compute_shoc_temperature(ComputeShocTempData& d) +{ + shoc_init(d.nlev, true, true); + d.transpose(); + compute_shoc_temperature_c(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); + d.transpose(); +} + // end _c impls // @@ -1118,7 +1132,9 @@ void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_se const auto thetal_zi_s = ekat::subview(thetal_zi_d, i); const auto w3_s = ekat::subview(w3_d, i); - SHF::compute_diag_third_shoc_moment(team, nlev, nlevi, w_sec_s, thl_sec_s, + // Hardcode runtime options for F90 testing + const Real c_diag_3rd_mom = 7.0; + SHF::compute_diag_third_shoc_moment(team, nlev, nlevi, c_diag_3rd_mom, w_sec_s, thl_sec_s, wthl_sec_s, tke_s, dz_zt_s, dz_zi_s, isotropy_zi_s, brunt_zi_s, w_sec_zi_s, thetal_zi_s, w3_s); }); @@ -1208,7 +1224,8 @@ void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, const auto zt_grid_s = ekat::subview(zt_grid_d, i); const auto shoc_mix_s = ekat::subview(shoc_mix_d, i); - SHF::compute_shoc_mix_shoc_length(team, nlev, tke_s, brunt_s, zt_grid_s, l_inf_s, + const Real length_fac = 0.5; + SHF::compute_shoc_mix_shoc_length(team, nlev, length_fac, tke_s, brunt_s, zt_grid_s, l_inf_s, shoc_mix_s); }); @@ -1511,6 +1528,11 @@ void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* q const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const Int i = team.league_rank(); + // Hardcode runtime options for F90 + const Real thl2tune = 1.0; + const Real qw2tune = 1.0; + const Real qwthl2tune = 1.0; + const Real w2tune = 1.0; const auto thetal_1d = ekat::subview(thetal_2d, i); const auto qw_1d = ekat::subview(qw_2d, i); @@ -1537,7 +1559,8 @@ void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* q const auto tkh_zi_1d = ekat::subview(tkh_zi_2d, i); const auto tk_zi_1d = ekat::subview(tk_zi_2d, i); - SHOC::diag_second_moments(team, nlev, nlevi, thetal_1d, qw_1d, u_wind_1d, v_wind_1d, tke_1d, isotropy_1d, tkh_1d, tk_1d, + SHOC::diag_second_moments(team, nlev, nlevi, thl2tune, qw2tune, qwthl2tune, w2tune, + thetal_1d, qw_1d, u_wind_1d, v_wind_1d, tke_1d, isotropy_1d, tkh_1d, tk_1d, dz_zi_1d, zt_grid_1d, zi_grid_1d, shoc_mix_1d, isotropy_zi_1d, tkh_zi_1d, tk_zi_1d, thl_sec_1d, qw_sec_1d, wthl_sec_1d, wqw_sec_1d, qwthl_sec_1d, uw_sec_1d, vw_sec_1d, wtke_sec_1d, w_sec_1d); @@ -1618,6 +1641,11 @@ void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Re Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const Int i = team.league_rank(); + // Hardcode runtime options for F90 + const Real thl2tune = 1.0; + const Real qw2tune = 1.0; + const Real qwthl2tune = 1.0; + const Real w2tune = 1.0; auto workspace = workspace_mgr.get_workspace(team); @@ -1651,6 +1679,7 @@ void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Re Scalar wstar_s = wstar_1d(i); SHOC::diag_second_shoc_moments(team, nlev, nlevi, + thl2tune, qw2tune, qwthl2tune, w2tune, thetal_1d, qw_1d, u_wind_1d, v_wind_1d, tke_1d, isotropy_1d, tkh_1d, tk_1d, dz_zi_1d, zt_grid_1d, zi_grid_1d, shoc_mix_1d, wthl_s, wqw_s, uw_s, vw_s, ustar2_s, wstar_s, workspace, thl_sec_1d, qw_sec_1d, wthl_sec_1d, wqw_sec_1d, qwthl_sec_1d, @@ -1953,7 +1982,9 @@ void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, const auto brunt_s = ekat::subview(brunt_d, i); const auto shoc_mix_s = ekat::subview(shoc_mix_d, i); - SHF::shoc_length(team,nlev,nlevi,host_dx_s,host_dy_s, + // Hardcode runtime option for F90 tests. + const Scalar length_fac = 0.5; + SHF::shoc_length(team,nlev,nlevi,length_fac,host_dx_s,host_dy_s, zt_grid_s,zi_grid_s,dz_zt_s,tke_s, thv_s,workspace,brunt_s,shoc_mix_s); }); @@ -2296,7 +2327,9 @@ void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real const auto zi_grid_s = ekat::subview(zi_grid_d, i); const auto w3_s = ekat::subview(w3_d, i); - SHF::diag_third_shoc_moments(team, nlev, nlevi, wsec_s, thl_sec_s, + // Hardcode for F90 testing + const Real c_diag_3rd_mom = 7.0; + SHF::diag_third_shoc_moments(team, nlev, nlevi, c_diag_3rd_mom, wsec_s, thl_sec_s, wthl_sec_s, isotropy_s, brunt_s, thetal_s, tke_s, dz_zt_s, dz_zi_s, zt_grid_s, zi_grid_s, workspace, @@ -2634,7 +2667,13 @@ void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, //outputs const auto isotropy_s = ekat::subview(isotropy_d, i); //output - SHF::isotropic_ts(team, nlev, brunt_int_s, tke_s, a_diss_s, brunt_s, isotropy_s); + // Hard code these runtime options for F90 + const Real lambda_low = 0.001; + const Real lambda_high = 0.04; + const Real lambda_slope = 2.65; + const Real lambda_thresh = 0.02; + SHF::isotropic_ts(team, nlev, lambda_low, lambda_high, lambda_slope, lambda_thresh, + brunt_int_s, tke_s, a_diss_s, brunt_s, isotropy_s); }); // Sync back to host @@ -2845,6 +2884,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, brunt_d, isotropy_d}; + SHF::SHOCRuntime shoc_runtime_options{0.001,0.04,2.65,0.02,1.0,1.0,1.0,1.0,0.5,7.0,0.1,0.1}; const auto nlevi_packs = ekat::npack(nlevi); @@ -2867,22 +2907,23 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, view_2d rho_zt ("rho_zt", shcol, nlevi_packs), shoc_qv ("shoc_qv", shcol, nlevi_packs), + tabs ("shoc_tabs", shcol, nlev_packs), dz_zt ("dz_zt", shcol, nlevi_packs), dz_zi ("dz_zi", shcol, nlevi_packs), tkhv ("tkh", shcol, nlevi_packs); SHF::SHOCTemporaries shoc_temporaries{ se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, - rho_zt, shoc_qv, dz_zt, dz_zi, tkhv}; + rho_zt, shoc_qv, tabs, dz_zt, dz_zi, tkhv}; #endif // Create local workspace const int n_wind_slots = ekat::npack(2)*Spack::n; const int n_trac_slots = ekat::npack(num_qtracers+3)*Spack::n; - ekat::WorkspaceManager workspace_mgr(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy); + ekat::WorkspaceManager workspace_mgr(nlevi_packs, 14+(n_wind_slots+n_trac_slots), policy); const auto elapsed_microsec = SHF::shoc_main(shcol, nlev, nlevi, npbl, nadv, num_qtracers, dtime, - workspace_mgr, + workspace_mgr, shoc_runtime_options, shoc_input, shoc_input_output, shoc_output, shoc_history_output #ifdef SCREAM_SMALL_KERNELS , shoc_temporaries @@ -3122,7 +3163,7 @@ void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, R ekat::device_to_host({dz_zt, dz_zi, rho_zt}, {shcol, shcol, shcol}, {nlev, nlevi, nlev}, inout_views, true); } -void eddy_diffusivities_f(Int nlev, Int shcol, Real* obklen, Real* pblh, Real* zt_grid, Real* shoc_mix, Real* sterm_zt, +void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, Real* isotropy, Real* tke, Real* tkh, Real* tk) { using SHF = Functions; @@ -3135,41 +3176,40 @@ void eddy_diffusivities_f(Int nlev, Int shcol, Real* obklen, Real* pblh, Real* z using ExeSpace = typename KT::ExeSpace; using MemberType = typename SHF::MemberType; - static constexpr Int num_1d_arrays = 2; - static constexpr Int num_2d_arrays = 7; + static constexpr Int num_1d_arrays = 1; + static constexpr Int num_2d_arrays = 8; std::vector temp_1d_d(num_1d_arrays); std::vector temp_2d_d(num_2d_arrays); - std::vector ptr_array = {zt_grid, shoc_mix, sterm_zt, isotropy, - tke, tkh, tk}; + std::vector ptr_array = {zt_grid, tabs, shoc_mix, sterm_zt, + isotropy, tke, tkh, tk}; // Sync to device - ScreamDeepCopy::copy_to_device({obklen, pblh}, shcol, temp_1d_d); + ScreamDeepCopy::copy_to_device({pblh}, shcol, temp_1d_d); ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d, true); - view_1d - obklen_d(temp_1d_d[0]), - pblh_d(temp_1d_d[1]); + view_1d pblh_d(temp_1d_d[0]); view_2d zt_grid_d(temp_2d_d[0]), - shoc_mix_d(temp_2d_d[1]), - sterm_zt_d(temp_2d_d[2]), - isotropy_d(temp_2d_d[3]), - tke_d(temp_2d_d[4]), - tkh_d(temp_2d_d[5]), - tk_d(temp_2d_d[6]); + tabs_d(temp_2d_d[1]), + shoc_mix_d(temp_2d_d[2]), + sterm_zt_d(temp_2d_d[3]), + isotropy_d(temp_2d_d[4]), + tke_d(temp_2d_d[5]), + tkh_d(temp_2d_d[6]), + tk_d(temp_2d_d[7]); const Int nk_pack = ekat::npack(nlev); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const Int i = team.league_rank(); - const Scalar obklen_s{obklen_d(i)}; const Scalar pblh_s{pblh_d(i)}; const auto zt_grid_s = ekat::subview(zt_grid_d, i); + const auto tabs_s = ekat::subview(tabs_d, i); const auto shoc_mix_s = ekat::subview(shoc_mix_d, i); const auto sterm_zt_s = ekat::subview(sterm_zt_d, i); const auto isotropy_s = ekat::subview(isotropy_d, i); @@ -3177,7 +3217,10 @@ void eddy_diffusivities_f(Int nlev, Int shcol, Real* obklen, Real* pblh, Real* z const auto tkh_s = ekat::subview(tkh_d, i); const auto tk_s = ekat::subview(tk_d, i); - SHF::eddy_diffusivities(team, nlev, obklen_s, pblh_s, zt_grid_s, shoc_mix_s, sterm_zt_s, isotropy_s, tke_s, tkh_s, tk_s); + // Hardcode runtime options for F90 testing + const Real Ckh = 0.1; + const Real Ckm = 0.1; + SHF::eddy_diffusivities(team, nlev, Ckh, Ckm, pblh_s, zt_grid_s, tabs_s, shoc_mix_s, sterm_zt_s, isotropy_s, tke_s, tkh_s, tk_s); }); // Sync back to host @@ -3359,7 +3402,7 @@ void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real } void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, - Real* u_wind, Real* v_wind, Real* brunt, Real* obklen, Real* zt_grid, Real* zi_grid, Real* pblh, Real* tke, Real* tk, + Real* tabs, Real* u_wind, Real* v_wind, Real* brunt, Real* zt_grid, Real* zi_grid, Real* pblh, Real* tke, Real* tk, Real* tkh, Real* isotropy) { using SHF = Functions; @@ -3372,25 +3415,23 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real using ExeSpace = typename KT::ExeSpace; using MemberType = typename SHF::MemberType; - static constexpr Int num_1d_arrays = 2; - static constexpr Int num_2d_arrays = 14; + static constexpr Int num_1d_arrays = 1; + static constexpr Int num_2d_arrays = 15; std::vector temp_1d_d(num_1d_arrays); std::vector temp_2d_d(num_2d_arrays); std::vector dim1_sizes(num_2d_arrays, shcol); - std::vector dim2_sizes = {nlev, nlev, nlev, nlev, nlevi, nlev, nlev, + std::vector dim2_sizes = {nlev, nlev, nlev, nlev, nlevi, nlev, nlev, nlev, nlev, nlev, nlevi, nlev, nlev, nlev, nlev}; - std::vector ptr_array = {wthv_sec, shoc_mix, u_wind, v_wind, dz_zi, dz_zt, pres, + std::vector ptr_array = {wthv_sec, shoc_mix, u_wind, v_wind, dz_zi, dz_zt, pres, tabs, brunt, zt_grid, zi_grid, tke, tk, tkh, isotropy}; // Sync to device - ScreamDeepCopy::copy_to_device({obklen, pblh}, shcol, temp_1d_d); + ScreamDeepCopy::copy_to_device({pblh}, shcol, temp_1d_d); ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); - view_1d - obklen_d(temp_1d_d[0]), - pblh_d(temp_1d_d[1]); + view_1d pblh_d(temp_1d_d[0]); view_2d wthv_sec_d(temp_2d_d[0]), @@ -3400,13 +3441,14 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real dz_zi_d(temp_2d_d[4]), dz_zt_d(temp_2d_d[5]), pres_d(temp_2d_d[6]), - brunt_d(temp_2d_d[7]), - zt_grid_d(temp_2d_d[8]), - zi_grid_d(temp_2d_d[9]), - tke_d(temp_2d_d[10]), - tk_d(temp_2d_d[11]), - tkh_d(temp_2d_d[12]), - isotropy_d(temp_2d_d[13]); + tabs_d(temp_2d_d[7]), + brunt_d(temp_2d_d[8]), + zt_grid_d(temp_2d_d[9]), + zi_grid_d(temp_2d_d[10]), + tke_d(temp_2d_d[11]), + tk_d(temp_2d_d[12]), + tkh_d(temp_2d_d[13]), + isotropy_d(temp_2d_d[14]); const Int nlev_packs = ekat::npack(nlev); const Int nlevi_packs = ekat::npack(nlevi); @@ -3420,7 +3462,6 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real auto workspace = workspace_mgr.get_workspace(team); - const Scalar obklen_s{obklen_d(i)}; const Scalar pblh_s{pblh_d(i)}; const auto wthv_sec_s = ekat::subview(wthv_sec_d, i); @@ -3430,6 +3471,7 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real const auto dz_zi_s = ekat::subview(dz_zi_d, i); const auto dz_zt_s = ekat::subview(dz_zt_d, i); const auto pres_s = ekat::subview(pres_d, i); + const auto tabs_s = ekat::subview(tabs_d, i); const auto brunt_s = ekat::subview(brunt_d, i); const auto zt_grid_s = ekat::subview(zt_grid_d, i); const auto zi_grid_s = ekat::subview(zi_grid_d, i); @@ -3438,8 +3480,18 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real const auto tkh_s = ekat::subview(tkh_d, i); const auto isotropy_s = ekat::subview(isotropy_d, i); - SHF::shoc_tke(team,nlev,nlevi,dtime,wthv_sec_s,shoc_mix_s,dz_zi_s,dz_zt_s,pres_s, - u_wind_s,v_wind_s,brunt_s,obklen_s,zt_grid_s,zi_grid_s,pblh_s, + // Hardcode for F90 testing + const Real lambda_low = 0.001; + const Real lambda_high = 0.04; + const Real lambda_slope = 2.65; + const Real lambda_thresh = 0.02; + const Real Ckh = 0.1; + const Real Ckm = 0.1; + + SHF::shoc_tke(team,nlev,nlevi,dtime,lambda_low,lambda_high,lambda_slope,lambda_thresh, + Ckh, Ckm, + wthv_sec_s,shoc_mix_s,dz_zi_s,dz_zt_s,pres_s, + tabs_s,u_wind_s,v_wind_s,brunt_s,zt_grid_s,zi_grid_s,pblh_s, workspace, tke_s,tk_s,tkh_s,isotropy_s); }); @@ -3449,5 +3501,46 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real ekat::device_to_host({tke, tk, tkh, isotropy}, shcol, nlev, inout_views, true); } +void compute_shoc_temperature_f(Int shcol, Int nlev, Real *thetal, Real *ql, Real *inv_exner, Real* tabs) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_2d = typename SHF::view_2d; + using KT = typename SHF::KT; + using ExeSpace = typename KT::ExeSpace; + using MemberType = typename SHF::MemberType; + + static constexpr Int num_arrays = 4; + + // Sync to device + std::vector temp_d(num_arrays); + ekat::host_to_device({thetal, ql, inv_exner, tabs}, shcol, nlev, temp_d, true); + + // Inputs/Outputs + view_2d + thetal_d(temp_d[0]), + ql_d(temp_d[1]), + inv_exner_d(temp_d[2]), + tabs_d(temp_d[3]); + + const Int nk_pack = ekat::npack(nlev); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nk_pack); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const Int i = team.league_rank(); + + const auto thetal_s = ekat::subview(thetal_d, i); + const auto ql_s = ekat::subview(ql_d, i); + const auto inv_exner_s = ekat::subview(inv_exner_d, i); + const auto tabs_s = ekat::subview(tabs_d, i); + + SHF::compute_shoc_temperature(team, nlev, thetal_s, ql_s, inv_exner_s, tabs_s); + }); + + // Sync back to host + std::vector out_views = {tabs_d}; + ekat::device_to_host({tabs}, shcol, nlev, out_views, true); +} + } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.hpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.hpp index 1e91d4b9e205..bc61efb168e6 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.hpp @@ -141,13 +141,13 @@ struct ShocEnergyTotalFixerData : public ShocTestGridDataBase { // Inputs Int shcol, nlev, nlevi, nadv; Real dtime; - Real *se_b, *ke_b, *wv_b, *wl_b, *se_a, *ke_a, *wv_a, *wl_a, *wthl_sfc, *wqw_sfc, *rho_zt; + Real *se_b, *ke_b, *wv_b, *wl_b, *se_a, *ke_a, *wv_a, *wl_a, *wthl_sfc, *wqw_sfc, *rho_zt, *pint; // Outputs Real *te_a, *te_b; ShocEnergyTotalFixerData(Int shcol_, Int nlev_, Int nlevi_, Real dtime_, Int nadv_) : - ShocTestGridDataBase({{ shcol_, nlev_ }, { shcol_, nlevi_ }, { shcol_ }}, {{ &zt_grid, &rho_zt }, { &zi_grid }, { &se_b, &ke_b, &wv_b, &wl_b, &se_a, &ke_a, &wv_a, &wl_a, &wthl_sfc, &wqw_sfc, &te_a, &te_b }}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_), nadv(nadv_), dtime(dtime_) {} + ShocTestGridDataBase({{ shcol_, nlev_ }, { shcol_, nlevi_ }, { shcol_ }}, {{ &zt_grid, &rho_zt }, { &zi_grid, &pint }, { &se_b, &ke_b, &wv_b, &wl_b, &se_a, &ke_a, &wv_a, &wl_a, &wthl_sfc, &wqw_sfc, &te_a, &te_b }}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_), nadv(nadv_), dtime(dtime_) {} PTD_STD_DEF(ShocEnergyTotalFixerData, 5, shcol, nlev, nlevi, dtime, nadv); }; @@ -314,7 +314,7 @@ struct ShocTkeData : public ShocTestGridDataBase { // Inputs Int shcol, nlev, nlevi; Real dtime; - Real *wthv_sec, *shoc_mix, *dz_zi, *dz_zt, *pres, *u_wind, *v_wind, *brunt, *obklen, *pblh; + Real *wthv_sec, *shoc_mix, *dz_zi, *dz_zt, *pres, *tabs, *u_wind, *v_wind, *brunt, *pblh; // Inputs/Outputs Real *tke, *tk, *tkh; @@ -323,7 +323,11 @@ struct ShocTkeData : public ShocTestGridDataBase { Real *isotropy; ShocTkeData(Int shcol_, Int nlev_, Int nlevi_, Real dtime_) : - ShocTestGridDataBase({{ shcol_, nlev_ }, { shcol_, nlevi_ }, { shcol_ }}, {{ &wthv_sec, &shoc_mix, &dz_zt, &pres, &u_wind, &v_wind, &brunt, &zt_grid, &tke, &tk, &tkh, &isotropy }, { &dz_zi, &zi_grid }, { &obklen, &pblh }}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_), dtime(dtime_) {} + ShocTestGridDataBase({{ shcol_, nlev_ }, { shcol_, nlevi_ }, { shcol_ }}, + {{ &wthv_sec, &shoc_mix, &dz_zt, &pres, &tabs, &u_wind, &v_wind, &brunt, &zt_grid, &tke, &tk, &tkh, &isotropy }, + { &dz_zi, &zi_grid }, + { &pblh }}), + shcol(shcol_), nlev(nlev_), nlevi(nlevi_), dtime(dtime_) {} PTD_STD_DEF(ShocTkeData, 4, shcol, nlev, nlevi, dtime); }; @@ -377,13 +381,13 @@ struct AdvSgsTkeData : public PhysicsTestData { struct EddyDiffusivitiesData : public PhysicsTestData { // Inputs Int shcol, nlev; - Real *obklen, *pblh, *zt_grid, *shoc_mix, *sterm_zt, *isotropy, *tke; + Real *pblh, *zt_grid, *tabs, *shoc_mix, *sterm_zt, *isotropy, *tke; // Outputs Real *tkh, *tk; EddyDiffusivitiesData(Int shcol_, Int nlev_) : - PhysicsTestData({{ shcol_ }, { shcol_, nlev_ }}, {{ &obklen, &pblh }, { &zt_grid, &shoc_mix, &sterm_zt, &isotropy, &tke, &tkh, &tk }}), shcol(shcol_), nlev(nlev_) {} + PhysicsTestData({{ shcol_ }, { shcol_, nlev_ }}, {{ &pblh }, { &zt_grid, &tabs, &shoc_mix, &sterm_zt, &isotropy, &tke, &tkh, &tk }}), shcol(shcol_), nlev(nlev_) {} PTD_STD_DEF(EddyDiffusivitiesData, 2, shcol, nlev); }; @@ -1051,6 +1055,20 @@ struct PblintdData : public PhysicsTestData { PTD_STD_DEF(PblintdData, 4, shcol, nlev, nlevi, npbl); }; +struct ComputeShocTempData : public PhysicsTestData { + // Inputs + Int shcol, nlev; + Real *thetal, *ql, *inv_exner; + + // Outputs + Real *tabs; + + ComputeShocTempData(Int shcol_, Int nlev_) : + PhysicsTestData({{ shcol_, nlev_ }}, {{ &thetal, &ql, &inv_exner, &tabs}}), shcol(shcol_), nlev(nlev_) {} + + PTD_STD_DEF(ComputeShocTempData, 2, shcol, nlev); +}; + // Glue functions to call fortran from from C++ with the Data struct void shoc_grid (ShocGridData& d); @@ -1119,6 +1137,7 @@ void vd_shoc_decomp_and_solve (VdShocDecompandSolveData& d void pblintd_surf_temp(PblintdSurfTempData& d); void pblintd_check_pblh(PblintdCheckPblhData& d); void pblintd(PblintdData& d); +void compute_shoc_temperature(ComputeShocTempData& d); extern "C" { // _f function decls void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, @@ -1219,11 +1238,12 @@ void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Rea void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh); void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt); -void eddy_diffusivities_f(Int nlev, Int shcol, Real* obklen, Real* pblh, Real* zt_grid, Real* shoc_mix, Real* sterm_zt, Real* isotropy, +void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, Real* isotropy, Real* tke, Real* tkh, Real* tk); void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, Real* u_wind, Real* v_wind, Real* brunt, Real* obklen, Real* zt_grid, Real* zi_grid, Real* pblh, Real* tke, Real* tk, Real* tkh, Real* isotropy); +void compute_shoc_temperature_f(Int shcol, Int nlev, Real* thetal, Real* ql, Real* inv_exner, Real* tabs); } // end _f function decls } // namespace shoc diff --git a/components/eamxx/src/physics/shoc/shoc_iso_c.f90 b/components/eamxx/src/physics/shoc/shoc_iso_c.f90 index e5f86fa2fe7e..438df4edadcd 100644 --- a/components/eamxx/src/physics/shoc/shoc_iso_c.f90 +++ b/components/eamxx/src/physics/shoc/shoc_iso_c.f90 @@ -24,7 +24,7 @@ subroutine append_precision(string, prefix) end subroutine append_precision subroutine shoc_init_c(nlev, gravit, rair, rh2o, cpair, & - zvir, latvap, latice, karman) bind(c) + zvir, latvap, latice, karman, p0) bind(c) use shoc, only: shoc_init, npbl integer(kind=c_int), value, intent(in) :: nlev ! number of levels @@ -37,20 +37,21 @@ subroutine shoc_init_c(nlev, gravit, rair, rh2o, cpair, & real(kind=c_real), value, intent(in) :: latvap ! latent heat of vaporization real(kind=c_real), value, intent(in) :: latice ! latent heat of fusion real(kind=c_real), value, intent(in) :: karman ! Von Karman's constant + real(kind=c_real), value, intent(in) :: p0 ! Reference pressure real(kind=c_real) :: pref_mid(nlev) ! unused values pref_mid = 0 call shoc_init(nlev, gravit, rair, rh2o, cpair, & - zvir, latvap, latice, karman, & + zvir, latvap, latice, karman, p0, & pref_mid, nlev, 1) npbl = nlev ! set pbl layer explicitly so we don't need pref_mid. end subroutine shoc_init_c ! shoc_init for shoc_main_bfb testing subroutine shoc_init_for_main_bfb_c(nlev, gravit, rair, rh2o, cpair, & - zvir, latvap, latice, karman,pref_mid,& - nbot_shoc, ntop_shoc) bind(c) + zvir, latvap, latice, karman, p0, & + pref_mid, nbot_shoc, ntop_shoc) bind(c) use shoc, only: shoc_init integer(kind=c_int), value, intent(in) :: nlev ! number of levels @@ -65,10 +66,11 @@ subroutine shoc_init_for_main_bfb_c(nlev, gravit, rair, rh2o, cpair, & real(kind=c_real), value, intent(in) :: latvap ! latent heat of vaporization real(kind=c_real), value, intent(in) :: latice ! latent heat of fusion real(kind=c_real), value, intent(in) :: karman ! Von Karman's constant + real(kind=c_real), value, intent(in) :: p0 ! Reference pressure real(kind=c_real), intent(in), dimension(nlev) :: pref_mid ! reference pressures at midpoints call shoc_init(nlev, gravit, rair, rh2o, cpair, & - zvir, latvap, latice, karman, & + zvir, latvap, latice, karman, p0, & pref_mid, nbot_shoc, ntop_shoc) end subroutine shoc_init_for_main_bfb_c @@ -292,7 +294,7 @@ subroutine check_tke_c(shcol, nlev, tke) bind(C) end subroutine check_tke_c subroutine shoc_tke_c(shcol, nlev, nlevi, dtime, wthv_sec, shoc_mix, dz_zi, & - dz_zt, pres, u_wind, v_wind, brunt, obklen, zt_grid, & + dz_zt, pres, tabs, u_wind, v_wind, brunt, zt_grid, & zi_grid, pblh, tke, tk, tkh, isotropy) bind(C) use shoc, only: shoc_tke @@ -305,10 +307,10 @@ subroutine shoc_tke_c(shcol, nlev, nlevi, dtime, wthv_sec, shoc_mix, dz_zi, & real(kind=c_real), intent(in) :: dz_zi(shcol,nlevi) real(kind=c_real), intent(in) :: dz_zt(shcol,nlev) real(kind=c_real), intent(in) :: pres(shcol,nlev) + real(kind=c_real), intent(in) :: tabs(shcol,nlev) real(kind=c_real), intent(in) :: u_wind(shcol,nlev) real(kind=c_real), intent(in) :: v_wind(shcol,nlev) real(kind=c_real), intent(in) :: brunt(shcol,nlev) - real(kind=c_real), intent(in) :: obklen(shcol) real(kind=c_real), intent(in) :: zt_grid(shcol,nlev) real(kind=c_real), intent(in) :: zi_grid(shcol,nlevi) real(kind=c_real), intent(in) :: pblh(shcol) @@ -319,7 +321,7 @@ subroutine shoc_tke_c(shcol, nlev, nlevi, dtime, wthv_sec, shoc_mix, dz_zi, & real(kind=c_real), intent(out) :: isotropy(shcol,nlev) call shoc_tke(shcol, nlev, nlevi, dtime, wthv_sec, shoc_mix, dz_zi, & - dz_zt, pres, u_wind, v_wind, brunt, obklen, zt_grid, & + dz_zt, pres, tabs, u_wind, v_wind, brunt, zt_grid, & zi_grid, pblh, tke, tk, tkh, isotropy) end subroutine shoc_tke_c @@ -392,15 +394,15 @@ subroutine adv_sgs_tke_c(nlev, shcol, dtime, shoc_mix, wthv_sec, & end subroutine adv_sgs_tke_c - subroutine eddy_diffusivities_c(nlev, shcol, obklen, pblh, zt_grid, & + subroutine eddy_diffusivities_c(nlev, shcol, pblh, zt_grid, tabs, & shoc_mix, sterm_zt, isotropy, tke, tkh, tk) bind (C) use shoc, only: eddy_diffusivities integer(kind=c_int), intent(in), value :: nlev integer(kind=c_int), intent(in), value :: shcol - real(kind=c_real), intent(in) :: obklen(shcol) real(kind=c_real), intent(in) :: pblh(shcol) real(kind=c_real), intent(in) :: zt_grid(shcol,nlev) + real(kind=c_real), intent(in) :: tabs(shcol,nlev) real(kind=c_real), intent(in) :: shoc_mix(shcol,nlev) real(kind=c_real), intent(in) :: sterm_zt(shcol,nlev) real(kind=c_real), intent(in) :: isotropy(shcol,nlev) @@ -409,7 +411,7 @@ subroutine eddy_diffusivities_c(nlev, shcol, obklen, pblh, zt_grid, & real(kind=c_real), intent(out) :: tkh(shcol,nlev) real(kind=c_real), intent(out) :: tk(shcol,nlev) - call eddy_diffusivities(nlev, shcol, obklen, pblh, zt_grid, & + call eddy_diffusivities(nlev, shcol, pblh, zt_grid, tabs, & shoc_mix, sterm_zt, isotropy, tke, tkh, tk) end subroutine eddy_diffusivities_c @@ -501,7 +503,7 @@ subroutine shoc_energy_total_fixer_c(& zt_grid,zi_grid,& se_b,ke_b,wv_b,wl_b,& se_a,ke_a,wv_a,wl_a,& - wthl_sfc,wqw_sfc,rho_zt,& + wthl_sfc,wqw_sfc,rho_zt,pint,& te_a,te_b) bind (C) use shoc, only: shoc_energy_total_fixer @@ -524,6 +526,7 @@ subroutine shoc_energy_total_fixer_c(& real(kind=c_real), intent(in) :: zt_grid(shcol,nlev) real(kind=c_real), intent(in) :: zi_grid(shcol,nlevi) real(kind=c_real), intent(in) :: rho_zt(shcol,nlev) + real(kind=c_real), intent(in) :: pint(shcol,nlevi) real(kind=c_real), intent(out) :: te_a(shcol) real(kind=c_real), intent(out) :: te_b(shcol) @@ -532,7 +535,7 @@ subroutine shoc_energy_total_fixer_c(& zt_grid,zi_grid,& se_b,ke_b,wv_b,wl_b,& se_a,ke_a,wv_a,wl_a,& - wthl_sfc,wqw_sfc,rho_zt,& + wthl_sfc,wqw_sfc,rho_zt,pint,& te_a,te_b) end subroutine shoc_energy_total_fixer_c @@ -1263,8 +1266,8 @@ subroutine diag_second_moments_lbycond_c(shcol, wthl_sfc, wqw_sfc, uw_sfc, vw_sf call diag_second_moments_lbycond(shcol, wthl_sfc, wqw_sfc, uw_sfc, vw_sfc, ustar2, wstar, wthl_sec, wqw_sec, uw_sec, vw_sec, wtke_sec, thl_sec, qw_sec, qwthl_sec) end subroutine diag_second_moments_lbycond_c - - subroutine diag_second_moments_c(shcol, nlev, nlevi, thetal, qw, u_wind, v_wind, tke, isotropy, tkh, tk, dz_zi, zt_grid, zi_grid, shoc_mix, thl_sec, qw_sec, & + + subroutine diag_second_moments_c(shcol, nlev, nlevi, thetal, qw, u_wind, v_wind, tke, isotropy, tkh, tk, dz_zi, zt_grid, zi_grid, shoc_mix, thl_sec, qw_sec, & wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec) bind(C) use shoc, only : diag_second_moments @@ -1278,7 +1281,7 @@ subroutine diag_second_moments_c(shcol, nlev, nlevi, thetal, qw, u_wind, v_wind, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec) end subroutine diag_second_moments_c - + subroutine diag_second_shoc_moments_c(shcol, nlev, nlevi, thetal, qw, u_wind, v_wind, tke, isotropy, tkh, tk, dz_zi, zt_grid, & zi_grid, shoc_mix, wthl_sfc, wqw_sfc, uw_sfc, vw_sfc, thl_sec, qw_sec, wthl_sec, wqw_sec, & qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec) bind(C) @@ -1379,7 +1382,7 @@ subroutine pblintd_surf_temp_c(shcol, nlev, nlevi, z, ustar, obklen, kbfs, thv, real(kind=c_real) , intent(inout), dimension(shcol) :: pblh logical(kind=c_bool) , intent(inout), dimension(shcol) :: check real(kind=c_real) , intent(inout), dimension(shcol, nlev) :: rino - + ! setup npbl npbl = nlev call pblintd_surf_temp(shcol, nlev, nlevi, z, ustar, obklen, kbfs, thv, tlv, pblh, check, rino) @@ -1412,5 +1415,15 @@ subroutine pblintd_c(shcol, nlev, nlevi, npbl_in, z, zi, thl, ql, q, u, v, ustar npbl = npbl_in call pblintd(shcol, nlev, nlevi, z, zi, thl, ql, q, u, v, ustar, obklen, kbfs, cldn, pblh) end subroutine pblintd_c + + subroutine compute_shoc_temperature_c(shcol, nlev, thetal, ql, inv_exner, tabs) bind(C) + use shoc, only : compute_shoc_temperature + + integer(kind=c_int) , value, intent(in) :: shcol, nlev + real(kind=c_real) , intent(in), dimension(shcol, nlev) :: thetal, ql, inv_exner + real(kind=c_real) , intent(out), dimension(shcol, nlev) :: tabs + + call compute_shoc_temperature(shcol, nlev, thetal, ql, inv_exner, tabs) + end subroutine compute_shoc_temperature_c end module shoc_iso_c diff --git a/components/eamxx/src/physics/shoc/shoc_iso_f.f90 b/components/eamxx/src/physics/shoc/shoc_iso_f.f90 index 7ae03d20b851..6cf55a9b4539 100644 --- a/components/eamxx/src/physics/shoc/shoc_iso_f.f90 +++ b/components/eamxx/src/physics/shoc/shoc_iso_f.f90 @@ -286,7 +286,7 @@ subroutine shoc_length_f(shcol, nlev, nlevi, host_dx, host_dy, & real(kind=c_real), intent(in) :: zt_grid(shcol,nlev) real(kind=c_real), intent(in) :: zi_grid(shcol,nlevi) real(kind=c_real), intent(in) :: dz_zt(shcol,nlev) - real(kind=c_real), intent(in) :: tke(shcol,nlev) + real(kind=c_real), intent(in) :: tke(shcol,nlev) real(kind=c_real), intent(in) :: thv(shcol,nlev) real(kind=c_real), intent(out) :: brunt(shcol,nlev) @@ -497,26 +497,36 @@ subroutine shoc_grid_f(shcol, nlev, nlevi, zt_grid, zi_grid, pdel, dz_zt, dz_zi, real(kind=c_real) , intent(out), dimension(shcol, nlevi) :: dz_zi end subroutine shoc_grid_f - subroutine eddy_diffusivities_f(nlev, shcol, obklen, pblh, zt_grid, shoc_mix, sterm_zt, isotropy, tke, tkh, tk) bind(C) + subroutine eddy_diffusivities_f(nlev, shcol, pblh, zt_grid, tabs, shoc_mix, sterm_zt, isotropy, tke, tkh, tk) bind(C) use iso_c_binding integer(kind=c_int) , value, intent(in) :: nlev, shcol - real(kind=c_real) , intent(in), dimension(shcol) :: obklen, pblh - real(kind=c_real) , intent(in), dimension(shcol, nlev) :: zt_grid, shoc_mix, sterm_zt, isotropy, tke + real(kind=c_real) , intent(in), dimension(shcol) :: pblh + real(kind=c_real) , intent(in), dimension(shcol, nlev) :: zt_grid, tabs, shoc_mix, sterm_zt, isotropy, tke real(kind=c_real) , intent(out), dimension(shcol, nlev) :: tkh, tk end subroutine eddy_diffusivities_f - subroutine shoc_tke_f(shcol, nlev, nlevi, dtime, wthv_sec, shoc_mix, dz_zi, dz_zt, pres, u_wind, v_wind, brunt, obklen, zt_grid, zi_grid, pblh, tke, tk, tkh, isotropy) bind(C) + subroutine shoc_tke_f(shcol, nlev, nlevi, dtime, wthv_sec, shoc_mix, dz_zi, dz_zt, pres, tabs, & + u_wind, v_wind, brunt, zt_grid, zi_grid, pblh, tke, tk, tkh, isotropy) bind(C) use iso_c_binding integer(kind=c_int) , value, intent(in) :: shcol, nlev, nlevi real(kind=c_real) , value, intent(in) :: dtime - real(kind=c_real) , intent(in), dimension(shcol, nlev) :: wthv_sec, shoc_mix, dz_zt, pres, u_wind, v_wind, brunt, zt_grid + real(kind=c_real) , intent(in), dimension(shcol, nlev) :: wthv_sec, shoc_mix, dz_zt, pres, tabs, u_wind, v_wind, brunt, zt_grid real(kind=c_real) , intent(in), dimension(shcol, nlevi) :: dz_zi, zi_grid - real(kind=c_real) , intent(in), dimension(shcol) :: obklen, pblh + real(kind=c_real) , intent(in), dimension(shcol) :: pblh real(kind=c_real) , intent(inout), dimension(shcol, nlev) :: tke, tk, tkh real(kind=c_real) , intent(out), dimension(shcol, nlev) :: isotropy end subroutine shoc_tke_f + + subroutine compute_shoc_temperature_f(shcol, nlev, thetal, ql, inv_exner, tabs) bind(C) + use iso_c_binding + + integer(kind=c_int) , value, intent(in) :: shcol, nlev + real(kind=c_real) , intent(in), dimension(shcol, nlev) :: thetal, ql, inv_exner + real(kind=c_real) , intent(out), dimension(shcol, nlev) :: tabs + end subroutine compute_shoc_temperature_f + end interface end module shoc_iso_f diff --git a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt index 0604b499d573..485f625e1423 100644 --- a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt @@ -1,7 +1,5 @@ -INCLUDE (ScreamUtils) +include (ScreamUtils) -SET (NEED_LIBS shoc physics_share scream_share) -SET (SK_NEED_LIBS shoc_sk physics_share scream_share) set(SHOC_TESTS_SRCS shoc_tests.cpp shoc_grid_tests.cpp @@ -70,31 +68,41 @@ set(SHOC_TESTS_SRCS shoc_pblintd_surf_temp_tests.cpp shoc_pblintd_check_pblh_tests.cpp shoc_pblintd_tests.cpp + shoc_compute_shoc_temperature_tests.cpp ) # SHOC_TESTS_SRCS # NOTE: tests inside this if statement won't be built in a baselines-only build if (NOT SCREAM_BASELINES_ONLY) - CreateUnitTest(shoc_tests "${SHOC_TESTS_SRCS}" "${NEED_LIBS}" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} DEP shoc_tests_ut_np1_omp1) + CreateUnitTest(shoc_tests "${SHOC_TESTS_SRCS}" + LIBS shoc + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + ) + if (NOT SCREAM_SMALL_KERNELS) - CreateUnitTest(shoc_sk_tests "${SHOC_TESTS_SRCS}" "${SK_NEED_LIBS}" THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} DEP shoc_tests_ut_np1_omp1 EXE_ARGS shoc_main_bfb) + CreateUnitTest(shoc_sk_tests "${SHOC_TESTS_SRCS}" + LIBS shoc_sk + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + EXE_ARGS shoc_main_bfb + ) endif() endif() if (SCREAM_ENABLE_BASELINE_TESTS) set(BASELINE_FILE_ARG "-b ${SCREAM_TEST_DATA_DIR}/shoc_run_and_cmp.baseline") - CreateUnitTestExec(shoc_run_and_cmp "shoc_run_and_cmp.cpp" "${NEED_LIBS}" - EXCLUDE_MAIN_CPP) + CreateUnitTestExec(shoc_run_and_cmp "shoc_run_and_cmp.cpp" + LIBS shoc + EXCLUDE_MAIN_CPP) CreateUnitTestFromExec(shoc_run_and_cmp_cxx shoc_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "shoc;physics") + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "shoc;physics") CreateUnitTestFromExec(shoc_run_and_cmp_f90 shoc_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f ${BASELINE_FILE_ARG}" - LABELS "shoc;physics") + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "-f ${BASELINE_FILE_ARG}" + LABELS "shoc;physics") # # Use fake tests to generate shell commands to generate baselines diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp new file mode 100644 index 000000000000..ae359978cf32 --- /dev/null +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp @@ -0,0 +1,274 @@ +#include "catch2/catch.hpp" + +#include "shoc_unit_tests_common.hpp" +#include "shoc_functions.hpp" +#include "shoc_functions_f90.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/scream_types.hpp" +#include "share/util/scream_setup_random_test.hpp" + +#include "ekat/ekat_pack.hpp" +#include "ekat/util/ekat_arch.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" + +#include +#include +#include +#include + +namespace scream { +namespace shoc { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestComputeShocTemp { + + static void run_property() + { + static constexpr Int shcol = 1; + static constexpr Int nlev = 3; + + // Test One + // Given Exner value = 1 and cloud liquid = 0 everywhere + // verify that all points of absolute temperature (tabs) + // are exactly equal to liquid water potential temperature (thetal) + + // Inverse Exner value [-] + Real inv_exner_first[nlev] = {1, 1, 1}; + // Liquid water potential temperature [K] + Real thetal_first[nlev] = {300, 290, 280}; + // Liquid water mixing ratio [kg/kg] + Real ql_first[nlev] = {0, 0, 0}; + + ComputeShocTempData SDS(shcol, nlev); + + REQUIRE(SDS.shcol > 0); + REQUIRE(SDS.nlev > 0); + + // Fill in test data on zt_grid. + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + SDS.inv_exner[offset] = inv_exner_first[n]; + SDS.thetal[offset] = thetal_first[n]; + SDS.ql[offset] = ql_first[n]; + } + } + + // Check that inputs are as expected + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.ql[offset] == 0); + REQUIRE(SDS.inv_exner[offset] == 1); + REQUIRE(SDS.thetal[offset] > 0); + } + } + + // Call the fortran implementation + compute_shoc_temperature(SDS); + + // Require that absolute temperature is equal to thetal + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.tabs[offset] == SDS.thetal[offset]); + } + } + + // Test Two + // Given profiles all with cloud liquid water greater than zero, + // AND inverse exner functions equal to 1, ensure that tabs is greater that + // absolute temperature is greater than (or equal to) liquid water potential temperature + + // Inverse Exner value [-] + Real inv_exner_sec[nlev] = {1, 1, 1}; + // Liquid water potential temperature [K] + Real thetal_sec[nlev] = {300, 290, 280}; + // Liquid water mixing ratio [kg/kg] + Real ql_sec[nlev] = {3e-5, 1e-10, 1e-3}; + + // Fill in test data on zt_grid. + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + SDS.inv_exner[offset] = inv_exner_sec[n]; + SDS.thetal[offset] = thetal_sec[n]; + SDS.ql[offset] = ql_sec[n]; + } + } + + // Check that inputs are as expected + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.ql[offset] > 0); + REQUIRE(SDS.inv_exner[offset] == 1); + REQUIRE(SDS.thetal[offset] > 0); + } + } + + // Call the fortran implementation + compute_shoc_temperature(SDS); + + // Require that absolute temperature is greather than thetal + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.tabs[offset] >= SDS.thetal[offset]); + } + } + + // Test Three + // Given "realistic" atmospheric profiles with thetal increasing + // with height, as well as inv_exner function; verify that + // absolute temperature is decreasing with height. Give + // all levels the same amount of cloud liquid water + + // Inverse Exner value [-] + Real inv_exner_third[nlev] = {1.1, 1.5, 2}; + // Liquid water potential temperature [K] + Real thetal_third[nlev] = {300, 350, 400}; + // Liquid water mixing ratio [kg/kg] + Real ql_third[nlev] = {1e-5, 1e-5, 1e-5}; + + // Fill in test data on zt_grid. + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + SDS.inv_exner[offset] = inv_exner_third[n]; + SDS.thetal[offset] = thetal_third[n]; + SDS.ql[offset] = ql_third[n]; + } + } + + // Check that inputs are as expected + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.ql[offset] > 0); + REQUIRE(SDS.inv_exner[offset] > 1); + REQUIRE(SDS.thetal[offset] > 0); + } + } + + // Check that inputs are changing with height as expected + for(Int s = 0; s < shcol; ++s) { + for(Int n = 1; n < nlev; ++n) { + const auto offset = n + s * nlev; + const auto offsetl = (n-1) + s * nlev; + + // Verify inverse exner and thetal are increasing with height + REQUIRE(SDS.inv_exner[offset] > SDS.inv_exner[offsetl]); + REQUIRE(SDS.thetal[offset] > SDS.thetal[offsetl]); + } + } + + // Call the fortran implementation + compute_shoc_temperature(SDS); + + // Require that absolute temperature be less than thetal + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.tabs[offset] < SDS.thetal[offset]); + } + } + + // Check that tabs is decreasing with height as expected + for(Int s = 0; s < shcol; ++s) { + for(Int n = 1; n < nlev; ++n) { + const auto offset = n + s * nlev; + const auto offsetl = (n-1) + s * nlev; + + REQUIRE(SDS.tabs[offset] < SDS.tabs[offsetl]); + } + } + + } + + static void run_bfb() + { + auto engine = setup_random_test(); + + ComputeShocTempData f90_data[] = { + // shcol, nlev + ComputeShocTempData(10, 71), + ComputeShocTempData(10, 12), + ComputeShocTempData(7, 16), + ComputeShocTempData(2, 7) + }; + + // Generate random input data + for (auto& d : f90_data) { + d.randomize(engine); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + ComputeShocTempData cxx_data[] = { + ComputeShocTempData(f90_data[0]), + ComputeShocTempData(f90_data[1]), + ComputeShocTempData(f90_data[2]), + ComputeShocTempData(f90_data[3]), + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + compute_shoc_temperature(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + d.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); + d.transpose(); // go back to C layout + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeShocTempData); + for (Int i = 0; i < num_runs; ++i) { + ComputeShocTempData& d_f90 = f90_data[i]; + ComputeShocTempData& d_cxx = cxx_data[i]; + for (Int k = 0; k < d_f90.total(d_f90.tabs); ++k) { + REQUIRE(d_f90.tabs[k] == d_cxx.tabs[k]); + } + } + } + } +}; + +} // namespace unit_test +} // namespace shoc +} // namespace scream + +namespace { + +TEST_CASE("shoc_compute_shoc_temperature_property", "shoc") + { + using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocTemp; + + TestStruct::run_property(); + } + +TEST_CASE("shoc_compute_shoc_temperature_bfb", "shoc") +{ + using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocTemp; + + TestStruct::run_bfb(); +} + +} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp index 7be0e91d4c37..4cb8d7a99c19 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp @@ -38,13 +38,13 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { // be loaded up with a different test. // FIRST TEST - // Boundary layer regime test. Input that have identical values - // except for the Monin Obukhov length, in this case will be positive - // and negative. Test to make sure that the resulting diffusivites - // are DIFFERENT + // Boundary layer regime test. Input surface temperature with a value that is clearly + // "running away" (i.e. a very low value). This test verifies that the diffusivities + // returned are different given that all other conditions are the same. This is to very + // that SHOC can repair a runaway surface temperature. - // Monin Obukov length [m] - static constexpr Real obklen_reg[shcol] = {-1, 1}; + // Surface temperature [K] + static constexpr Real tabs[shcol] = {100, 250}; // PBL depth [m] static constexpr Real pblh = 1000; // zt_grid [m] @@ -69,7 +69,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { for(Int s = 0; s < shcol; ++s) { // Column only input SDS.pblh[s] = pblh; - SDS.obklen[s] = obklen_reg[s]; + SDS.tabs[s] = tabs[s]; for(Int n = 0; n < nlev; ++n) { const auto offset = n + s * nlev; @@ -112,13 +112,13 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } // SECOND TEST - // Stable boundary layer test. Given Obukhov length, + // Runaway stable boundary layer test. Given Obukhov length, // verify that each regime behaves as expected when the relevant // inputs are modified. Column with larger mixing length and // shear term should produce larger diffusivity values. - // Monin Obukov length [m] - static constexpr Real obklen_stab[shcol] = {1, 1}; + // Surface temperature [K] + static constexpr Real tabs_stab[shcol] = {100, 100}; // SHOC Mixing length [m] static constexpr Real shoc_mix_stab[shcol] = {100, 150}; // Shear term [s-2] @@ -139,7 +139,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { // Fill in test data on zt_grid. for(Int s = 0; s < shcol; ++s) { // Column only input - SDS.obklen[s] = obklen_stab[s]; + SDS.tabs[s] = tabs_stab[s]; for(Int n = 0; n < nlev; ++n) { const auto offset = n + s * nlev; @@ -152,8 +152,8 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { // Check that the inputs make sense for(Int s = 0; s < shcol; ++s) { - // Make sure we are testing stable boundary layer - REQUIRE(SDS.obklen[s] > 0); + // Make sure we are testing a runaway stable boundary layer + REQUIRE(SDS.tabs[s] < 180); for (Int n = 0; n < nlev; ++n){ const auto offset = n + s * nlev; // Should be greater than zero @@ -183,12 +183,12 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } // THIRD TEST - // Unstable boundary layer test. Given Obukhov length, + // Default boundary layer test. // verify that each regime behaves as expected when the relevant - // inputs are modified. + // inputs are modified for the default definitions of diffusivities. - // Monin Obukov length [m] - static constexpr Real obklen_ustab[shcol] = {-1, -1}; + // Surface temperature [K] + static constexpr Real tabs_ustab[shcol] = {300, 300}; // SHOC Mixing length [m] static constexpr Real shoc_mix_ustab = 500; // Shear term [s-2] @@ -209,7 +209,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { // Fill in test data on zt_grid. for(Int s = 0; s < shcol; ++s) { // Column only input - SDS.obklen[s] = obklen_ustab[s]; + SDS.tabs[s] = tabs_ustab[s]; for(Int n = 0; n < nlev; ++n) { const auto offset = n + s * nlev; @@ -222,8 +222,8 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { // Check that the inputs make sense for(Int s = 0; s < shcol; ++s) { - // Make sure we are testing unstable boundary layer - REQUIRE(SDS.obklen[s] < 0); + // Make sure we are testing default boundary layer diffusivities + REQUIRE(SDS.tabs[s] > 220); for (Int n = 0; n < nlev; ++n){ const auto offset = n + s * nlev; // Should be greater than zero @@ -253,6 +253,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } + static void run_bfb() { auto engine = setup_random_test(); @@ -290,7 +291,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { // Get data from cxx for (auto& d : cxx_data) { d.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(d.nlev, d.shcol, d.obklen, d.pblh, d.zt_grid, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); + eddy_diffusivities_f(d.nlev, d.shcol, d.pblh, d.zt_grid, d.tabs, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); d.transpose(); // go back to C layout } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp index 408c2cb69b41..ea59b217f775 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp @@ -55,6 +55,8 @@ struct UnitWrap::UnitTest::TestShocTotEnergyFixer { static constexpr Real wthl_sfc = 0.5; // Define surface total water flux [kg/kg m/s] static constexpr Real wqw_sfc = 0.01; + // Pressure at interface [Pa] + static constexpr Real pint[nlevi] = {50000, 60000, 70000, 80000, 90000, 100000}; // Initialize data structure for bridging to F90 ShocEnergyTotalFixerData SDS(shcol, nlev, nlevi, dtime, nadv); @@ -93,6 +95,7 @@ struct UnitWrap::UnitTest::TestShocTotEnergyFixer { const auto offset = n + s * nlevi; SDS.zi_grid[offset] = zi_grid[n]; + SDS.pint[offset] = pint[n]; } } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp index 3e4d264c56c4..bc975c9b59c1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp @@ -57,8 +57,8 @@ struct UnitWrap::UnitTest::TestShocTke { Real u_wind[nlev] = {2, 1, 0, -1, -2}; // Define meridional wind on nlev grid [m/s] Real v_wind[nlev] = {1, 2, 3, 4, 5}; - // Define Obukov length [m] - Real obklen = -1; + // Define surface temperature [K] (value irrelevant, just make sure it's physical) + Real tabs[nlev] ={300, 300, 300, 300, 300}; // Define thickness on the interface grid [m] Real dz_zi[nlevi] = {0, 100, 100, 100, 100, 50}; // Define thickness on the thermo grid [m] @@ -94,7 +94,6 @@ struct UnitWrap::UnitTest::TestShocTke { // Fill in test data on zt_grid. for(Int s = 0; s < shcol; ++s) { SDS.pblh[s] = pblh; - SDS.obklen[s] = obklen; for(Int n = 0; n < nlev; ++n) { const auto offset = n + s * nlev; @@ -108,6 +107,7 @@ struct UnitWrap::UnitTest::TestShocTke { SDS.tke[offset] = tke_init[n]; SDS.tkh[offset] = tkh[n]; SDS.tk[offset] = tk[n]; + SDS.tabs[offset] = tabs[n]; } // Fill in test data on zi_grid. @@ -244,7 +244,6 @@ struct UnitWrap::UnitTest::TestShocTke { REQUIRE(SDS.isotropy[offset] <= maxiso); } } - } static void run_bfb() @@ -285,7 +284,7 @@ struct UnitWrap::UnitTest::TestShocTke { for (auto& d : cxx_data) { d.transpose(); // _f expects data in fortran layout shoc_tke_f(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, - d.pres, d.u_wind, d.v_wind, d.brunt, d.obklen, d.zt_grid, d.zi_grid, d.pblh, + d.pres, d.tabs, d.u_wind, d.v_wind, d.brunt, d.zt_grid, d.zi_grid, d.pblh, d.tke, d.tk, d.tkh, d.isotropy); d.transpose(); // go back to C layout } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp b/components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp index 97962c16ca6a..0d74ebaa5fe1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp @@ -119,6 +119,7 @@ struct UnitWrap { struct TestPblintdSurfTemp; struct TestPblintdCheckPblh; struct TestPblintd; + struct TestComputeShocTemp; }; }; diff --git a/components/eamxx/src/physics/spa/CMakeLists.txt b/components/eamxx/src/physics/spa/CMakeLists.txt index f267ab0b12ed..26d2ba86e203 100644 --- a/components/eamxx/src/physics/spa/CMakeLists.txt +++ b/components/eamxx/src/physics/spa/CMakeLists.txt @@ -1,4 +1,5 @@ -add_library(spa atmosphere_prescribed_aerosol.cpp) +add_library(spa eamxx_spa_process_interface.cpp) +target_compile_definitions(spa PUBLIC EAMXX_HAS_SPA) target_link_libraries(spa physics_share scream_share) if (NOT SCREAM_LIB_ONLY) diff --git a/components/eamxx/src/physics/spa/atmosphere_prescribed_aerosol.cpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp similarity index 99% rename from components/eamxx/src/physics/spa/atmosphere_prescribed_aerosol.cpp rename to components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp index 53a113f469e1..6b59ce16a877 100644 --- a/components/eamxx/src/physics/spa/atmosphere_prescribed_aerosol.cpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp @@ -1,4 +1,4 @@ -#include "atmosphere_prescribed_aerosol.hpp" +#include "eamxx_spa_process_interface.hpp" #include "share/util/scream_time_stamp.hpp" #include "share/io/scream_scorpio_interface.hpp" diff --git a/components/eamxx/src/physics/spa/atmosphere_prescribed_aerosol.hpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp similarity index 100% rename from components/eamxx/src/physics/spa/atmosphere_prescribed_aerosol.hpp rename to components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index c36a1ded3cf2..665f4537083c 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -431,8 +431,13 @@ ::update_spa_data_from_file( spa_data_in_params.set("Filename",spa_data_file_name); spa_data_in_params.set("Skip_Grid_Checks",true); // We need to skip grid checks because multiple ranks may want the same column of source data. + // Retrieve number of cols on spa_data_file. + scorpio::register_file(spa_data_file_name,scorpio::Read); + int num_global_cols = scorpio::get_dimlen(spa_data_file_name,"ncol"); + scorpio::eam_pio_closefile(spa_data_file_name); + // Construct the grid needed for input: - auto grid = std::make_shared("grid",num_local_cols,source_data_nlevs,comm); + auto grid = std::make_shared("grid",num_local_cols,num_global_cols,source_data_nlevs,comm); Kokkos::deep_copy(grid->get_dofs_gids().template get_view(),unique_src_dofs); grid->get_dofs_gids().sync_to_host(); diff --git a/components/eamxx/src/physics/spa/tests/CMakeLists.txt b/components/eamxx/src/physics/spa/tests/CMakeLists.txt index e23c8fb491b2..effe97702505 100644 --- a/components/eamxx/src/physics/spa/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/spa/tests/CMakeLists.txt @@ -1,17 +1,18 @@ INCLUDE (ScreamUtils) -SET (NEED_LIBS spa scream_io physics_share scream_share) - -CreateUnitTest(spa_read_data_test "spa_read_data_from_file_test.cpp" "${NEED_LIBS}" - LABELS "spa" +CreateUnitTest(spa_read_data_test "spa_read_data_from_file_test.cpp" + LIBS spa scream_io + LABELS spa MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) -CreateUnitTest(spa_one_to_one_remap_test "spa_one_to_one_remap_test.cpp" "${NEED_LIBS}" - LABELS "spa" +CreateUnitTest(spa_one_to_one_remap_test "spa_one_to_one_remap_test.cpp" + LIBS spa scream_io + LABELS spa MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) -CreateUnitTest(spa_main_test "spa_main_test.cpp" "${NEED_LIBS}" - LABELS "spa" +CreateUnitTest(spa_main_test "spa_main_test.cpp" + LIBS spa scream_io + LABELS spa ) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/spa_main.yaml diff --git a/components/eamxx/src/physics/tms/CMakeLists.txt b/components/eamxx/src/physics/tms/CMakeLists.txt new file mode 100644 index 000000000000..b6e1715e5f32 --- /dev/null +++ b/components/eamxx/src/physics/tms/CMakeLists.txt @@ -0,0 +1,30 @@ +# Create tms library +add_library(tms eamxx_tms_process_interface.cpp) +target_link_libraries(tms physics_share scream_share) +target_compile_definitions(tms PUBLIC EAMXX_HAS_TMS) +target_include_directories(tms PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/impl +) + +# Add ETI source files if not on CUDA/HIP +if (NOT EAMXX_ENABLE_GPU) + target_sources(tms PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/eti/compute_tms.cpp + ) +endif() + +if (NOT SCREAM_LIB_ONLY) + # For testing, add some more sources and modules directory + target_sources (tms PUBLIC + ${SCREAM_BASE_DIR}/../eam/src/physics/cam/trb_mtn_stress.F90 + ${CMAKE_CURRENT_SOURCE_DIR}/tms_iso_c.f90 + ${CMAKE_CURRENT_SOURCE_DIR}/tms_functions_f90.cpp + ) + set_target_properties(tms PROPERTIES + Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tms_modules + ) + target_include_directories (tms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/tms_modules) + + add_subdirectory(tests) +endif() diff --git a/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp b/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp new file mode 100644 index 000000000000..c06b87a966b1 --- /dev/null +++ b/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp @@ -0,0 +1,177 @@ +#include "eamxx_tms_process_interface.hpp" + +#include "physics/tms/tms_functions.hpp" + +#include "ekat/ekat_assert.hpp" +#include "ekat/util/ekat_units.hpp" + +#include + +namespace scream +{ + +// ========================================================================================= +TurbulentMountainStress::TurbulentMountainStress (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereProcess(comm, params) +{ + // Do nothing +} + +// ========================================================================================= +void TurbulentMountainStress::set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + // Define some useful units. The units of mixing ratio + // Q are technically non-dimensional. Nevertheless, + // for output reasons, we like to see 'kg/kg'. + auto Qunit = kg/kg; + Qunit.set_string("kg/kg"); + const auto nondim = Units::nondimensional(); + const auto m2 = m*m; + + // Initialize grid from grids manager + m_grid = grids_manager->get_grid("Physics"); + const auto& grid_name = m_grid->name(); + EKAT_REQUIRE_MSG(grid_name=="Physics PG2", + "Error! TMS process can only be used with \"Physics PG2\" physics grid. " + "Current physics grid is "+grid_name+".\n"); + + m_ncols = m_grid->get_num_local_dofs(); // Number of columns on this rank + m_nlevs = m_grid->get_num_vertical_levels(); // Number of levels per column + + // Add required/computed fields + FieldLayout scalar2d_layout{ {COL}, {m_ncols} }, + scalar3d_midpoint_layout{ {COL, LEV}, {m_ncols, m_nlevs} }, + vector2d_layout{ {COL, CMP}, {m_ncols, 2} }, + vector3d_midpoint_layout{ {COL, CMP, LEV}, {m_ncols, 2, m_nlevs} }; + + constexpr int ps = Spack::n; + add_field("horiz_winds", vector3d_midpoint_layout, m/s, grid_name, ps); + add_field("T_mid", scalar3d_midpoint_layout, K, grid_name, ps); + add_field("p_mid", scalar3d_midpoint_layout, Pa, grid_name, ps); + add_field("pseudo_density", scalar3d_midpoint_layout, Pa, grid_name, ps); + add_field("qv", scalar3d_midpoint_layout, Qunit, grid_name, "tracers", ps); + add_field("sgh30", scalar2d_layout, m, grid_name); + add_field("landfrac", scalar2d_layout, nondim, grid_name); + + add_field("surf_drag_coeff_tms", scalar2d_layout, kg/s/m2, grid_name); + add_field("wind_stress_tms", vector2d_layout, N/m2, grid_name); +} + +// ========================================================================================= +void TurbulentMountainStress::initialize_impl (const RunType /* run_type */) +{ + // Do nothing +} + +// ========================================================================================= +void TurbulentMountainStress::run_impl (const double /* dt */) +{ + // Helper views + const auto pseudo_density = get_field_in("pseudo_density").get_view(); + const auto qv = get_field_in("qv").get_view(); + const auto dz = m_buffer.dz; + const auto z_int = m_buffer.z_int; + + // Input views + const auto horiz_winds = get_field_in("horiz_winds").get_view(); + const auto T_mid = get_field_in("T_mid").get_view(); + const auto p_mid = get_field_in("p_mid").get_view(); + const auto sgh30 = get_field_in("sgh30").get_view(); + const auto landfrac = get_field_in("landfrac").get_view(); + const auto exner = m_buffer.exner; + const auto z_mid = m_buffer.z_mid; + + // Output views + const auto surf_drag_coeff_tms = get_field_out("surf_drag_coeff_tms").get_view(); + const auto wind_stress_tms = get_field_out("wind_stress_tms").get_view(); + + // Preprocess inputs + const int ncols = m_ncols; + const int nlevs = m_nlevs; + const int nlev_packs = ekat::npack(nlevs); + // calculate_z_int contains a team-level parallel_scan, which requires a special policy + const auto scan_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(ncols, nlev_packs); + Kokkos::parallel_for(scan_policy, KOKKOS_LAMBDA (const TMSFunctions::KT::MemberType& team) { + const int i = team.league_rank(); + + const auto p_mid_i = ekat::subview(p_mid, i); + const auto exner_i = ekat::subview(exner, i); + const auto pseudo_density_i = ekat::subview(pseudo_density, i); + const auto T_mid_i = ekat::subview(T_mid, i); + const auto qv_i = ekat::subview(qv, i); + const auto dz_i = ekat::subview(dz, i); + const auto z_int_i = ekat::subview(z_int, i); + const auto z_mid_i = ekat::subview(z_mid, i); + + // Calculate exner + PF::exner_function(team, p_mid_i, exner_i); + + // Calculate z_mid + PF::calculate_dz(team, pseudo_density_i, p_mid_i, T_mid_i, qv_i, dz_i); + const Real z_surf = 0.0; // For now, set z_int(i,nlevs) = z_surf = 0 + team.team_barrier(); + PF::calculate_z_int(team, nlevs, dz_i, z_surf, z_int_i); + team.team_barrier(); + PF::calculate_z_mid(team, nlevs, z_int_i, z_mid_i); + }); + + // Compute TMS + TMSFunctions::compute_tms(ncols, nlevs, + ekat::scalarize(horiz_winds), + ekat::scalarize(T_mid), + ekat::scalarize(p_mid), + ekat::scalarize(exner), + ekat::scalarize(z_mid), + sgh30, landfrac, + surf_drag_coeff_tms, wind_stress_tms); +} + +// ========================================================================================= +void TurbulentMountainStress::finalize_impl() +{ + // Do nothing +} +// ========================================================================================= +size_t TurbulentMountainStress::requested_buffer_size_in_bytes() const +{ + const int nlev_packs = ekat::npack(m_nlevs); + const int nlevi_packs = ekat::npack(m_nlevs+1); + return Buffer::num_2d_midpoint_views*m_ncols*nlev_packs*sizeof(Spack) + + Buffer::num_2d_interface_views*m_ncols*nlevi_packs*sizeof(Spack); +} +// ========================================================================================= +void TurbulentMountainStress::init_buffers(const ATMBufferManager &buffer_manager) +{ + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Buffers size not sufficient.\n"); + + Spack* mem = reinterpret_cast(buffer_manager.get_memory()); + const int nlev_packs = ekat::npack(m_nlevs); + const int nlevi_packs = ekat::npack(m_nlevs+1); + + uview_2d* buffer_mid_view_ptrs[Buffer::num_2d_midpoint_views] = { + &m_buffer.exner, &m_buffer.dz, &m_buffer.z_mid + }; + + for (int i=0; isize(); + } + + uview_2d* buffer_int_view_ptrs[Buffer::num_2d_interface_views] = { + &m_buffer.z_int + }; + + for (int i=0; isize(); + } + + size_t used_mem = (reinterpret_cast(mem) - buffer_manager.get_memory())*sizeof(Real); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for TurbulentMountainStress."); +} +} // namespace scream diff --git a/components/eamxx/src/physics/tms/eamxx_tms_process_interface.hpp b/components/eamxx/src/physics/tms/eamxx_tms_process_interface.hpp new file mode 100644 index 000000000000..ef140d1b76cf --- /dev/null +++ b/components/eamxx/src/physics/tms/eamxx_tms_process_interface.hpp @@ -0,0 +1,83 @@ +#ifndef SCREAM_TMS_PROCESS_HPP +#define SCREAM_TMS_PROCESS_HPP + +#include "physics/tms/tms_functions.hpp" +#include "share/atm_process/atmosphere_process.hpp" +#include "share/util/scream_common_physics_functions.hpp" +#include "ekat/ekat_parameter_list.hpp" + +#include + +namespace scream +{ + +/* + * The class responsible for computing/applying the surface drag coefficient + * and stress associated with subgrid mountains + * + * The AD should store exactly ONE instance of this class stored + * in its list of subcomponents (the AD should make sure of this). +*/ + +class TurbulentMountainStress : public AtmosphereProcess +{ + using PF = scream::PhysicsFunctions; + using TMSFunctions = tms::Functions; + using Spack = ekat::Pack; + using view_2d = TMSFunctions::view_2d; + using uview_2d = ekat::Unmanaged; + +public: + + // Constructors + TurbulentMountainStress (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The type of subcomponent + AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name () const { return "tms"; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + + // Structure for storing local variables initialized using the ATMBufferManager + struct Buffer { + static constexpr int num_2d_midpoint_views = 3; + static constexpr int num_2d_interface_views = 1; + + uview_2d exner, dz, z_mid, z_int; + }; + +#ifndef KOKKOS_ENABLE_CUDA + // Cuda requires methods enclosing __device__ lambda's to be public +protected: +#endif + + void run_impl (const double dt); + +protected: + + void initialize_impl (const RunType run_type); + void finalize_impl (); + + // Computes total number of bytes needed for local variables + size_t requested_buffer_size_in_bytes() const; + + // Set local variables using memory provided by + // the ATMBufferManager + void init_buffers(const ATMBufferManager &buffer_manager); + + // Struct which contains local variables + Buffer m_buffer; + + // Keep track of field dimensions and the iteration count + int m_ncols; + int m_nlevs; + + std::shared_ptr m_grid; +}; // class TurbulentMountainStress + +} // namespace scream + +#endif // SCREAM_TMS_PROCESS_HPP diff --git a/components/eamxx/src/physics/tms/eti/compute_tms.cpp b/components/eamxx/src/physics/tms/eti/compute_tms.cpp new file mode 100644 index 000000000000..8f0f16696a7b --- /dev/null +++ b/components/eamxx/src/physics/tms/eti/compute_tms.cpp @@ -0,0 +1,13 @@ +#include "compute_tms_impl.hpp" + +namespace scream { +namespace tms { + +/* + * Explicit instantiation using the default device. + */ + +template struct Functions; + +} // namespace tms +} // namespace scream diff --git a/components/eamxx/src/physics/tms/impl/compute_tms_impl.hpp b/components/eamxx/src/physics/tms/impl/compute_tms_impl.hpp new file mode 100644 index 000000000000..3ab7b8844beb --- /dev/null +++ b/components/eamxx/src/physics/tms/impl/compute_tms_impl.hpp @@ -0,0 +1,93 @@ +#ifndef COMPUTE_TMS_IMPL_HPP +#define COMPUTE_TMS_IMPL_HPP + +#include "tms_functions.hpp" // for ETI only but harmless for GPU + +namespace scream { +namespace tms { + +template +void Functions::compute_tms( + const int& ncols, + const int& nlevs, + const view_3d& horiz_wind, + const view_2d& t_mid, + const view_2d& p_mid, + const view_2d& exner, + const view_2d& z_mid, + const view_1d& sgh, + const view_1d& landfrac, + const view_1d& ksrf, + const view_2d& tau_tms) +{ + using C = physics::Constants; + + // Define some constants used + const Scalar horomin = 1; // Minimum value of subgrid orographic height for mountain stress [ m ] + const Scalar z0max = 100; // Maximum value of z_0 for orography [ m ] + const Scalar dv2min = 0.01; // Minimum shear squared [ m2/s2 ] + const Scalar orocnst = C::orocnst; // Converts from standard deviation to height [ no unit ] + const Scalar z0fac = C::z0fac; // Factor determining z_0 from orographic standard deviation [ no unit ] + const Scalar karman = C::Karman; // von Karman constant + const Scalar gravit = C::gravit; // Acceleration due to gravity + const Scalar rair = C::Rair; // Gas constant for dry air + + // Loop over columns + const typename KT::RangePolicy policy (0,ncols); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const int& i) { + // Subview on column, scalarize since we only care about last 2 levels (never loop over levels) + const auto u_wind_i = ekat::subview(horiz_wind, i, 0); + const auto v_wind_i = ekat::subview(horiz_wind, i, 1); + const auto t_mid_i = ekat::subview(t_mid, i); + const auto p_mid_i = ekat::subview(p_mid, i); + const auto exner_i = ekat::subview(exner, i); + const auto z_mid_i = ekat::subview(z_mid, i); + + // Determine subgrid orgraphic height (mean to peak) + const auto horo = orocnst*sgh(i); + + if (horo < horomin) { + // No mountain stress if horo is too small + ksrf(i) = 0; + tau_tms(i, 0) = 0; + tau_tms(i, 1) = 0; + } else { + // Determine z0m for orography + const auto z0oro = ekat::impl::min(z0fac*horo, z0max); + + // Calculate neutral drag coefficient + const auto tmp = karman/std::log((z_mid_i(nlevs-1) + z0oro )/z0oro); + auto cd = tmp*tmp; + + // Calculate the Richardson number over the lowest 2 layers + const auto kb = nlevs-1; + const auto kt = nlevs-2; + const auto tmp_u = u_wind_i(kt) - u_wind_i(kb); + const auto tmp_v = v_wind_i(kt) - v_wind_i(kb); + const auto dv2 = ekat::impl::max(tmp_u*tmp_u + tmp_v*tmp_v, dv2min); + + const auto ri = 2*gravit*(t_mid_i(kt)*exner_i(kt) - t_mid_i(kb)*exner_i(kb))*(z_mid_i(kt) - z_mid_i(kb))/ + ((t_mid_i(kt)*exner_i(kt) + t_mid_i(kb)*exner_i(kb))*dv2); + + // Calculate the instability function and modify the neutral drag cofficient. + // We should probably follow more elegant approach like Louis et al (1982) or + // Bretherton and Park (2009) but for now we use very crude approach : just 1 + // for ri < 0, 0 for ri > 1, and linear ramping. + const auto stabfri = ekat::impl::max(0.0, ekat::impl::min(1.0, 1.0-ri)); + cd *= stabfri; + + // Compute density, velocity magnitude and stress using bottom level properties + const auto rho = p_mid_i(nlevs-1)/(rair*t_mid_i(nlevs-1)); + const auto vmag = std::sqrt(u_wind_i(nlevs-1)*u_wind_i(nlevs-1) + + v_wind_i(nlevs-1)*v_wind_i(nlevs-1)); + ksrf(i) = rho*cd*vmag*landfrac(i); + tau_tms(i, 0) = -ksrf(i)*u_wind_i(nlevs-1); + tau_tms(i, 1) = -ksrf(i)*v_wind_i(nlevs-1); + } + }); +} + +} // namespace scream +} // namespace tms + +#endif // COMPUTE_TMS_IMPL_HPP diff --git a/components/eamxx/src/physics/tms/tests/CMakeLists.txt b/components/eamxx/src/physics/tms/tests/CMakeLists.txt new file mode 100644 index 000000000000..19cab9c53e21 --- /dev/null +++ b/components/eamxx/src/physics/tms/tests/CMakeLists.txt @@ -0,0 +1,9 @@ +INCLUDE (ScreamUtils) + +# NOTE: tests inside this if statement won't be built in a baselines-only build +if (NOT SCREAM_BASELINES_ONLY) + CreateUnitTest(tms_tests compute_tms_tests.cpp + LIBS tms + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + ) +endif() diff --git a/components/eamxx/src/physics/tms/tests/compute_tms_tests.cpp b/components/eamxx/src/physics/tms/tests/compute_tms_tests.cpp new file mode 100644 index 000000000000..ad6296524134 --- /dev/null +++ b/components/eamxx/src/physics/tms/tests/compute_tms_tests.cpp @@ -0,0 +1,106 @@ +#include "catch2/catch.hpp" + +#include "tms_unit_tests_common.hpp" + +#include "share/scream_types.hpp" +#include "ekat/ekat_pack.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "tms_functions.hpp" +#include "tms_functions_f90.hpp" +#include "share/util/scream_setup_random_test.hpp" + +namespace scream { +namespace tms { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestComputeTMS { + + static void run_property() + { + // Should property tests be created? + } // run_property + + static void run_bfb() + { + auto engine = setup_random_test(); + + ComputeTMSData f90_data[] = { + // ncols, nlevs + ComputeTMSData(12, 72), + ComputeTMSData(8, 12), + ComputeTMSData(7, 16), + ComputeTMSData(2, 7) + }; + + // Generate random input data + for (auto& d : f90_data) { + d.randomize(engine, { {d.sgh, {0.5, 1.5}} }); + } + + // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // inout data is in original state + ComputeTMSData cxx_data[] = { + ComputeTMSData(f90_data[0]), + ComputeTMSData(f90_data[1]), + ComputeTMSData(f90_data[2]), + ComputeTMSData(f90_data[3]) + }; + + // Assume all data is in C layout + + // Get data from fortran + for (auto& d : f90_data) { + // expects data in C layout + compute_tms(d); + } + + // Get data from cxx + for (auto& d : cxx_data) { + d.transpose(); // _f expects data in fortran layout + compute_tms_f(d.ncols, d.nlevs, + d.u_wind, d.v_wind, d.t_mid, d.p_mid, d.exner, + d.z_mid, d.sgh, d.landfrac, d.ksrf, d.taux, d.tauy); + d.transpose(); // go back to C layout + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING) { + static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeTMSData); + + for (int r = 0; r::TestComputeTMS; +// TestStruct::run_property(); +//} + +TEST_CASE("compute_tms_bfb", "tms") +{ + using TestStruct = scream::tms::unit_test::UnitWrap::UnitTest::TestComputeTMS; + TestStruct::run_bfb(); +} + +} // empty namespace diff --git a/components/eamxx/src/physics/tms/tests/tms_unit_tests_common.hpp b/components/eamxx/src/physics/tms/tests/tms_unit_tests_common.hpp new file mode 100644 index 000000000000..c6df67f694d1 --- /dev/null +++ b/components/eamxx/src/physics/tms/tests/tms_unit_tests_common.hpp @@ -0,0 +1,58 @@ +#ifndef TMS_UNIT_TESTS_COMMON_HPP +#define TMS_UNIT_TESTS_COMMON_HPP + +#include "tms_functions.hpp" +#include "share/scream_types.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" + +namespace scream { +namespace tms { +namespace unit_test { + +/* + * Unit test infrastructure for tms unit tests. + * + * tms entities can friend scream::tms::unit_test::UnitWrap to give unit tests + * access to private members. + * + * All unit test impls should be within an inner struct of UnitWrap::UnitTest for + * easy access to useful types. + */ + +struct UnitWrap { + + template + struct UnitTest : public KokkosTypes { + + using Device = D; + using MemberType = typename KokkosTypes::MemberType; + using TeamPolicy = typename KokkosTypes::TeamPolicy; + using RangePolicy = typename KokkosTypes::RangePolicy; + using ExeSpace = typename KokkosTypes::ExeSpace; + + template + using view_1d = typename KokkosTypes::template view_1d; + template + using view_2d = typename KokkosTypes::template view_2d; + template + using view_3d = typename KokkosTypes::template view_3d; + + template + using uview_1d = typename ekat::template Unmanaged >; + + using Functions = scream::tms::Functions; + using Scalar = typename Functions::Scalar; + using Spack = ekat::Pack; + + // Put struct decls here + struct TestComputeTMS; + }; + +}; + + +} // namespace unit_test +} // namespace tms +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/tms/tms_functions.hpp b/components/eamxx/src/physics/tms/tms_functions.hpp new file mode 100644 index 000000000000..b61291c79745 --- /dev/null +++ b/components/eamxx/src/physics/tms/tms_functions.hpp @@ -0,0 +1,69 @@ +#ifndef TMS_FUNCTIONS_HPP +#define TMS_FUNCTIONS_HPP + +#include "physics/share/physics_constants.hpp" + +#include "share/scream_types.hpp" + +#include "ekat/kokkos/ekat_subview_utils.hpp" +#include "ekat/ekat_pack_kokkos.hpp" +#include "ekat/ekat_workspace.hpp" + +namespace scream { +namespace tms { + +/* + * Functions is a stateless struct used to encapsulate a + * number of functions for a process. Currently only one + * function exists: compute_tms(). + */ + +template +struct Functions +{ + // + // ------- Types -------- + // + + using Scalar = ScalarT; + using Device = DeviceT; + + using KT = ekat::KokkosTypes; + + template + using view_1d = typename KT::template view_1d; + template + using view_2d = typename KT::template view_2d; + template + using view_3d = typename KT::template view_3d; + + // + // --------- Functions --------- + // + static void compute_tms( + const int& ncols, + const int& nlevs, + const view_3d& horiz_wind, + const view_2d& t_mid, + const view_2d& p_mid, + const view_2d& exner, + const view_2d& z_mid, + const view_1d& sgh, + const view_1d& landfrac, + const view_1d& ksrf, + const view_2d& tau_tms); + +}; // struct tms + +} // namespace tms +} // namespace scream + +// If a GPU build, without relocatable device code enabled, make all code available +// to the translation unit; otherwise, ETI is used. +#if defined(EAMXX_ENABLE_GPU) && !defined(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) \ + && !defined(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) + +# include "compute_tms_impl.hpp" +#endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE + +#endif // TMS_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/tms/tms_functions_f90.cpp b/components/eamxx/src/physics/tms/tms_functions_f90.cpp new file mode 100644 index 000000000000..f1aa2e771778 --- /dev/null +++ b/components/eamxx/src/physics/tms/tms_functions_f90.cpp @@ -0,0 +1,114 @@ +#include "tms_functions_f90.hpp" + +#include "ekat/ekat_assert.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "ekat/ekat_pack_kokkos.hpp" +#include "ekat/kokkos/ekat_subview_utils.hpp" + +#include "share/util/scream_deep_copy.hpp" + +#include + +using scream::Real; + +// +// A C interface to TMS fortran calls. The stubs below will link to fortran definitions in tms_iso_c.f90 +// +extern "C" { +void init_tms_c(Real orocnst, Real z0fac, Real karman, Real gravit, Real rair); +void compute_tms_c(int ncols, int nlevs, Real *u_wind, Real *v_wind, Real *t_mid, Real *p_mid, Real *exner, + Real *zm, Real *sgh, Real *landfrac, Real *ksrf, Real *taux, Real *tauy); +} + +namespace scream { +namespace tms { + +// Glue functions to call fortran from from C++ with the Data struct +void compute_tms(ComputeTMSData& d) +{ + using C = scream::physics::Constants; + init_tms_c(C::orocnst, C::z0fac, C::Karman, C::gravit, C::Rair); + d.transpose(); + compute_tms_c(d.ncols, d.nlevs, d.u_wind, d.v_wind, d.t_mid, d.p_mid, d.exner, + d.z_mid, d.sgh, d.landfrac, d.ksrf, d.taux, d.tauy); + d.transpose(); +} + +// +// _f function definitions. These expect data in C layout +// +void compute_tms_f(int ncols, int nlevs, + Real *u_wind, Real *v_wind, Real *t_mid, Real *p_mid, Real *exner, Real *z_mid, + Real *sgh, Real *landfrac, Real *ksrf, Real *taux, Real *tauy) +{ + using TMSFunc = Functions; + + using Scalar = typename TMSFunc::Scalar; + using Spack = ekat::Pack; + using view_1d = typename TMSFunc::view_1d; + using view_2d = typename TMSFunc::view_2d; + using view_2d_s = typename TMSFunc::view_2d; + using view_3d = typename TMSFunc::view_3d; + using ExeSpace = typename TMSFunc::KT::ExeSpace; + using MemberType = typename TMSFunc::KT::MemberType; + + // Initialize Kokkos views, sync to device + std::vector temp_d_1d(2); + std::vector temp_d_2d(6); + ScreamDeepCopy::copy_to_device({sgh, landfrac}, ncols, temp_d_1d); + ekat::host_to_device({u_wind, v_wind, t_mid, p_mid, exner, z_mid}, + ncols, nlevs, temp_d_2d, true); + + view_1d + sgh_d (temp_d_1d[0]), + landfrac_d(temp_d_1d[1]), + ksrf_d ("ksrf_d", ncols), + taux_d ("taux_d", ncols), + tauy_d ("tauy_d", ncols); + + view_2d + u_wind_d(temp_d_2d[0]), + v_wind_d(temp_d_2d[1]), + t_mid_d (temp_d_2d[2]), + p_mid_d (temp_d_2d[3]), + exner_d (temp_d_2d[4]), + z_mid_d (temp_d_2d[5]); + + // calculate_tms treats u/v_wind and taux/y as multiple component arrays. + const auto nlev_packs = ekat::npack(nlevs); + view_3d horiz_wind_d("horiz_wind_d", ncols, 2, nlev_packs); + view_2d_s tau_d("tau_d", ncols, 2); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, nlev_packs); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int i = team.league_rank(); + tau_d(i, 0) = taux_d(i); + tau_d(i, 1) = tauy_d(i); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_packs), [&] (const Int& k) { + horiz_wind_d(i,0,k) = u_wind_d(i,k); + horiz_wind_d(i,1,k) = v_wind_d(i,k); + }); + }); + + // C++ compute_tms function implementation + TMSFunc::compute_tms(ncols, nlevs, + ekat::scalarize(horiz_wind_d), + ekat::scalarize(t_mid_d), + ekat::scalarize(p_mid_d), + ekat::scalarize(exner_d), + ekat::scalarize(z_mid_d), + sgh_d, landfrac_d, ksrf_d, tau_d); + + // Transfer data back to individual arrays (only for output variables) + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int i = team.league_rank(); + taux_d(i) = tau_d(i, 0); + tauy_d(i) = tau_d(i, 1); + }); + + // Sync back to host + std::vector output_data = {ksrf_d, taux_d, tauy_d}; + ScreamDeepCopy::copy_to_host({ksrf, taux, tauy}, ncols, output_data); +} + +} // namespace tms +} // namespace scream diff --git a/components/eamxx/src/physics/tms/tms_functions_f90.hpp b/components/eamxx/src/physics/tms/tms_functions_f90.hpp new file mode 100644 index 000000000000..9176aa52cf12 --- /dev/null +++ b/components/eamxx/src/physics/tms/tms_functions_f90.hpp @@ -0,0 +1,53 @@ +#ifndef SCREAM_TMS_FUNCTIONS_F90_HPP +#define SCREAM_TMS_FUNCTIONS_F90_HPP + +#include "share/scream_types.hpp" +#include "physics/share/physics_test_data.hpp" + +#include "tms_functions.hpp" +#include "physics_constants.hpp" + +#include +#include +#include + +// +// Bridge functions to call fortran version of tms functions from C++ +// + +namespace scream { +namespace tms { + +struct ComputeTMSData : public PhysicsTestData +{ + // Input + int ncols, nlevs; + Real *u_wind, *v_wind, *t_mid, *p_mid, *exner, *z_mid, *sgh, *landfrac; + + // Output + Real *ksrf, *taux, *tauy; + + ComputeTMSData(int ncols_, int nlevs_) + : PhysicsTestData({ {ncols_}, {ncols_, nlevs_} }, + { {&sgh, &landfrac, &ksrf, &taux, &tauy}, + {&u_wind, &v_wind, &t_mid, &p_mid, &exner, &z_mid} }), + ncols(ncols_), nlevs(nlevs_) + {} + + PTD_STD_DEF(ComputeTMSData, 2, ncols, nlevs); +}; + +// Glue functions to call fortran from from C++ with the Data struct +void compute_tms(ComputeTMSData& d); + +// _f function decls +extern "C" { +void compute_tms_f(int ncols, int nlevs, + Real *u_wind, Real *v_wind, Real *t_mid, Real *p_mid, Real *exner, Real *z_mid, + Real *sgh, Real *landfrac, Real *ksrf, Real *taux, Real *tauy); +} // end _f function decls + +} // namespace tms +} // namespace scream + +#endif // SCREAM_TMS_FUNCTIONS_F90_HPP diff --git a/components/eamxx/src/physics/tms/tms_iso_c.f90 b/components/eamxx/src/physics/tms/tms_iso_c.f90 new file mode 100644 index 000000000000..8f5602da98ff --- /dev/null +++ b/components/eamxx/src/physics/tms/tms_iso_c.f90 @@ -0,0 +1,36 @@ + +module tms_iso_c + use iso_c_binding + implicit none + +#include "scream_config.f" + +! +! This file contains bridges from scream c++ to tms fortran. +! + +contains + subroutine init_tms_c(orocnst, z0fac, karman, gravit, rair) bind(c) + use trb_mtn_stress, only: init_tms + + real(kind=c_double), value, intent(in) :: orocnst, z0fac, karman, gravit, rair + character(len=128) :: errstring + + integer, parameter :: r8 = selected_real_kind(12) ! 8 byte real + + call init_tms(r8, orocnst, z0fac, karman, gravit, rair, errstring) + end subroutine init_tms_c + + subroutine compute_tms_c(ncols, nlevs, u_wind, v_wind, t_mid, p_mid, exner, & + zm, sgh, landfrac, ksrf, taux, tauy) bind(c) + use trb_mtn_stress, only: compute_tms + + integer(kind=c_int), value, intent(in) :: ncols, nlevs + real(kind=c_double) , intent(in), dimension(ncols, nlevs) :: u_wind,v_wind,t_mid,p_mid,exner,zm + real(kind=c_double) , intent(in), dimension(ncols) :: sgh,landfrac + real(kind=c_double) , intent(out), dimension(ncols) :: ksrf, taux, tauy + + call compute_tms(ncols, nlevs, ncols, u_wind, v_wind, t_mid, p_mid, exner, zm, sgh, ksrf, taux, tauy, landfrac) + end subroutine compute_tms_c + +end module tms_iso_c diff --git a/components/eamxx/src/physics/zm/CMakeLists.txt b/components/eamxx/src/physics/zm/CMakeLists.txt index 3a61fe728d1a..9867681599fc 100644 --- a/components/eamxx/src/physics/zm/CMakeLists.txt +++ b/components/eamxx/src/physics/zm/CMakeLists.txt @@ -2,13 +2,13 @@ set(ZM_SRCS ${SCREAM_BASE_DIR}/../eam/src/physics/cam/physics_utils.F90 ${SCREAM_BASE_DIR}/../eam/src/physics/cam/scream_abortutils.F90 zm_conv.F90 - atmosphere_deep_convection.cpp + eamxx_zm_process_interface.cpp scream_zm_interface.F90 ) set(ZM_HEADERS zm.hpp - atmosphere_deep_convection.hpp + eamxx_zm_process_interface.hpp scream_zm_interface.hpp ) @@ -23,7 +23,6 @@ set_target_properties(zm PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules ) target_include_directories(zm PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../common ${CMAKE_CURRENT_BINARY_DIR}/modules ) target_link_libraries(zm physics_share scream_share) diff --git a/components/eamxx/src/physics/zm/atmosphere_deep_convection.cpp b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp similarity index 99% rename from components/eamxx/src/physics/zm/atmosphere_deep_convection.cpp rename to components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp index f6120859e9e5..2dbb048957e6 100644 --- a/components/eamxx/src/physics/zm/atmosphere_deep_convection.cpp +++ b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp @@ -1,5 +1,5 @@ #include "physics/zm/scream_zm_interface.hpp" -#include "physics/zm/atmosphere_deep_convection.hpp" +#include "physics/zm/eamxx_zm_process_interface.hpp" #include "ekat/ekat_assert.hpp" diff --git a/components/eamxx/src/physics/zm/atmosphere_deep_convection.hpp b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.hpp similarity index 100% rename from components/eamxx/src/physics/zm/atmosphere_deep_convection.hpp rename to components/eamxx/src/physics/zm/eamxx_zm_process_interface.hpp diff --git a/components/eamxx/src/scream_config.h.in b/components/eamxx/src/scream_config.h.in index b21c677bd55b..77b1bf04ff0c 100644 --- a/components/eamxx/src/scream_config.h.in +++ b/components/eamxx/src/scream_config.h.in @@ -22,9 +22,6 @@ // Whether this is a CUDA/HIP build #cmakedefine EAMXX_ENABLE_GPU -// Whether SCREAM has Homme enabled as dynamics dycore -#cmakedefine SCREAM_HAS_HOMME - // Whether scream uses leap years or not #cmakedefine SCREAM_HAS_LEAP_YEAR @@ -51,4 +48,10 @@ // Whether monolithic kernels are on #cmakedefine SCREAM_SMALL_KERNELS +// The sha of the last commit +#define EAMXX_GIT_VERSION "${EAMXX_GIT_VERSION}" + +// The version of EAMxx +#define EAMXX_VERSION "${EAMXX_VERSION_MAJOR}.${EAMXX_VERSION_MINOR}.${EAMXX_VERSION_PATCH}" + #endif diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index 19f69ee2c45b..01236f1c57fe 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -18,21 +18,35 @@ set(SHARE_SRC field/field_manager.cpp grid/abstract_grid.cpp grid/grids_manager.cpp + grid/grid_import_export.cpp grid/se_grid.cpp grid/point_grid.cpp grid/remap/abstract_remapper.cpp grid/remap/coarsening_remapper.cpp + grid/remap/horiz_interp_remapper_base.cpp + grid/remap/refining_remapper_p2p.cpp grid/remap/vertical_remapper.cpp grid/remap/horizontal_remap_utility.cpp property_checks/property_check.cpp property_checks/field_nan_check.cpp property_checks/field_within_interval_check.cpp property_checks/mass_and_energy_column_conservation_check.cpp + util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp util/scream_time_stamp.cpp util/scream_timing.cpp util/scream_utils.cpp + util/eamxx_time_interpolation.cpp + util/scream_bfbhash.cpp + util/eamxx_time_interpolation.cpp ) +if (EAMXX_ENABLE_EXPERIMENTAL_CODE) + list (APPEND + SHARE_SRC + grid/remap/refining_remapper_rma.cpp + ) +endif() + add_library(scream_share ${SHARE_SRC}) set_target_properties(scream_share PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules @@ -40,9 +54,10 @@ set_target_properties(scream_share PROPERTIES target_include_directories(scream_share PUBLIC ${SCREAM_SRC_DIR} ${SCREAM_BIN_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/modules ) -target_link_libraries(scream_share PUBLIC ekat gptl) +target_link_libraries(scream_share PUBLIC ekat pioc) target_compile_options(scream_share PUBLIC $<$:${SCREAM_Fortran_FLAGS}> ) diff --git a/components/eamxx/src/share/atm_process/atmosphere_diagnostic.cpp b/components/eamxx/src/share/atm_process/atmosphere_diagnostic.cpp index fc778f252e0b..c7e08a90940e 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_diagnostic.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_diagnostic.cpp @@ -22,8 +22,6 @@ void AtmosphereDiagnostic::compute_diagnostic (const double dt) { // Some diagnostics need the timestep, store in case. m_dt = dt; - compute_diagnostic_impl (); - // Set the timestamp of the diagnostic to the most // recent timestamp among the inputs const auto& inputs = get_fields_in(); @@ -41,17 +39,50 @@ void AtmosphereDiagnostic::compute_diagnostic (const double dt) { " - Diag name: " + name() + "\n"); m_diagnostic_output.get_header().get_tracking().update_time_stamp(ts); -} + // Note: call the impl method *after* setting the diag time stamp. + // Some derived classes may "refuse" to compute the diag, due to some + // inconsistency of data. In that case, they can reset the diag time stamp + // to something invalid, which can be used by downstream classes to determine + // if the diag has been successfully computed or not. + compute_diagnostic_impl (); +} void AtmosphereDiagnostic::run_impl (const double dt) { compute_diagnostic(dt); } -void AtmosphereDiagnostic::set_computed_field (const Field& /* f */) { + +void AtmosphereDiagnostic:: +set_required_field_impl (const Field& f) { + // Check that the field has the pack size that was requested + // TODO: I don't think diagnostics should "request" a pack size. + // Diags should work with whatever the AD is storing. + // That's b/c the field is already allocated by the time + // we create any diagnostic. + // While we fix all diags, this method will at least + // throw an error if the pack size that the diag "requested" + // is not compatible with the field alloc props. + for (const auto& it : get_required_field_requests()) { + if (it.fid.name()==f.name()) { + const auto& fap = f.get_header().get_alloc_properties(); + EKAT_REQUIRE_MSG (fap.get_largest_pack_size()>=it.pack_size, + "Error! Diagnostic input field cannot accommodate the needed pack size.\n" + " - diag field: " + m_diagnostic_output.name() + "\n" + " - input field: " + f.name() + "\n" + " - requested pack size: " + std::to_string(it.pack_size) + "\n" + " - field max pack size: " + std::to_string(fap.get_largest_pack_size()) + "\n"); + break; + } + } +} + +void AtmosphereDiagnostic:: +set_computed_field_impl (const Field& /* f */) { EKAT_ERROR_MSG("Error! Diagnostics are not allowed to compute fields. See " + name() + ".\n"); } -void AtmosphereDiagnostic::set_computed_group (const FieldGroup& /* group */) { +void AtmosphereDiagnostic:: +set_computed_group_impl (const FieldGroup& /* group */) { EKAT_ERROR_MSG("Error! Diagnostics are not allowed to compute field groups. See " + name() + ".\n"); } diff --git a/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp b/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp index 789b05e0ce45..2bd7fc4ddf26 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp @@ -51,12 +51,13 @@ class AtmosphereDiagnostic : public AtmosphereProcess // Getting the diagnostic output Field get_diagnostic () const; - void set_computed_field (const Field& f) final; - void set_computed_group (const FieldGroup& group) final; - void compute_diagnostic (const double dt = 0); protected: + void set_required_field_impl (const Field& f) final; + void set_computed_field_impl (const Field& f) final; + void set_computed_group_impl (const FieldGroup& group) final; + virtual void compute_diagnostic_impl () = 0; // By default, diagnostic don't do any initialization/finalization stuff. diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.cpp b/components/eamxx/src/share/atm_process/atmosphere_process.cpp index 6d0c4c72d1f4..189c980c431c 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.cpp @@ -61,6 +61,8 @@ AtmosphereProcess (const ekat::Comm& comm, const ekat::ParameterList& params) // Info for mass and energy conservation checks m_column_conservation_check_data.has_check = m_params.get("enable_column_conservation_checks", false); + + m_internal_diagnostics_level = m_params.get("internal_diagnostics_level", 0); } void AtmosphereProcess::initialize (const TimeStamp& t0, const RunType run_type) { @@ -70,12 +72,21 @@ void AtmosphereProcess::initialize (const TimeStamp& t0, const RunType run_type) set_fields_and_groups_pointers(); m_time_stamp = t0; initialize_impl(run_type); + + // Create all start-of-step fields needed for tendencies calculation + for (const auto& it : m_proc_tendencies) { + const auto& tname = it.first; + const auto& fname = m_tend_to_field.at(tname); + m_start_of_step_fields[fname] = get_field_out(fname).clone(); + } + if (this->type()!=AtmosphereProcessType::Group) { stop_timer (m_timer_prefix + this->name() + "::init"); } } void AtmosphereProcess::run (const double dt) { + m_atm_logger->debug("[EAMxx::" + this->name() + "] run..."); start_timer (m_timer_prefix + this->name() + "::run"); if (m_params.get("enable_precondition_checks", true)) { // Run 'pre-condition' property checks stored in this AP @@ -97,9 +108,17 @@ void AtmosphereProcess::run (const double dt) { compute_column_conservation_checks_data(dt_sub); } + if (m_internal_diagnostics_level > 0) + print_global_state_hash(name() + "-pre-sc-" + std::to_string(m_subcycle_iter), + true, false, false); + // Run derived class implementation run_impl(dt_sub); + if (m_internal_diagnostics_level > 0) + print_global_state_hash(name() + "-pst-sc-" + std::to_string(m_subcycle_iter), + true, true, true); + if (has_column_conservation_check()) { // Run the column local mass and energy conservation checks run_column_conservation_check(); @@ -128,8 +147,8 @@ void AtmosphereProcess::finalize (/* what inputs? */) { void AtmosphereProcess::setup_tendencies_requests () { using vos_t = std::vector; - auto tend_vec = m_params.get("compute_tendencies",{"NONE"}); - if (tend_vec == vos_t{"NONE"}) { + auto tend_vec = m_params.get("compute_tendencies",{}); + if (tend_vec.size()==0) { return; } @@ -192,6 +211,7 @@ void AtmosphereProcess::setup_tendencies_requests () { // field with the requested name. If more than one is found (must be // that we have same field on multiple grids), error out. using namespace ekat::units; + using strlist_t = std::list; for (const auto& tn : tend_list) { std::string grid_found = ""; auto tokens = field_grid(tn); @@ -221,7 +241,7 @@ void AtmosphereProcess::setup_tendencies_requests () { // Create tend FID and request field FieldIdentifier t_fid(tname,layout,units,gname,dtype); - add_field(t_fid,"ACCUMULATED"); + add_field(t_fid,strlist_t{"ACCUMULATED","DIVIDE_BY_DT"}); grid_found = gname; } } @@ -344,6 +364,7 @@ void AtmosphereProcess::set_computed_group (const FieldGroup& group) { void AtmosphereProcess::run_property_check (const prop_check_ptr& property_check, const CheckFailHandling check_fail_handling, const PropertyCheckCategory property_check_category) const { + m_atm_logger->trace("[" + this->name() + "] run_property_check '" + property_check->name() + "'..."); auto res_and_msg = property_check->check(); // string for output @@ -360,7 +381,8 @@ void AtmosphereProcess::run_property_check (const prop_check_ptr& property "WARNING: Failed and repaired " + pre_post_str + " property check.\n" " - Atmosphere process name: " + name() + "\n" " - Property check name: " + property_check->name() + "\n" - " - Atmosphere process MPI Rank: " + std::to_string(m_comm.rank()) + "\n"); + " - Atmosphere process MPI Rank: " + std::to_string(m_comm.rank()) + "\n" + " - Message: " + res_and_msg.msg + "\n"); } else { // Ugh, the test failed badly, with no chance to repair it. if (check_fail_handling==CheckFailHandling::Warning) { @@ -435,54 +457,69 @@ void AtmosphereProcess::run_property_check (const prop_check_ptr& property } void AtmosphereProcess::run_precondition_checks () const { + m_atm_logger->debug("[" + this->name() + "] run_precondition_checks..."); + start_timer(m_timer_prefix + this->name() + "::run-precondition-checks"); // Run all pre-condition property checks for (const auto& it : m_precondition_checks) { run_property_check(it.second, it.first, PropertyCheckCategory::Precondition); } + stop_timer(m_timer_prefix + this->name() + "::run-precondition-checks"); + m_atm_logger->debug("[" + this->name() + "] run_precondition_checks...done!"); } void AtmosphereProcess::run_postcondition_checks () const { + m_atm_logger->debug("[" + this->name() + "] run_postcondition_checks..."); + start_timer(m_timer_prefix + this->name() + "::run-postcondition-checks"); // Run all post-condition property checks for (const auto& it : m_postcondition_checks) { run_property_check(it.second, it.first, PropertyCheckCategory::Postcondition); } + stop_timer(m_timer_prefix + this->name() + "::run-postcondition-checks"); + m_atm_logger->debug("[" + this->name() + "] run_postcondition_checks...done!"); } void AtmosphereProcess::run_column_conservation_check () const { + m_atm_logger->debug("[" + this->name() + "] run_column_conservation_check..."); + start_timer(m_timer_prefix + this->name() + "::run-column-conservation-checks"); // Conservation check is run as a postcondition check run_property_check(m_column_conservation_check.second, m_column_conservation_check.first, PropertyCheckCategory::Postcondition); + stop_timer(m_timer_prefix + this->name() + "::run-column-conservation-checks"); + m_atm_logger->debug("[" + this->name() + "] run_column-conservation_checks...done!"); } void AtmosphereProcess::init_step_tendencies () { if (m_compute_proc_tendencies) { start_timer(m_timer_prefix + this->name() + "::compute_tendencies"); - for (auto& it : m_proc_tendencies) { - const auto& tname = it.first; - const auto& fname = m_tend_to_field.at(tname); + for (auto& it : m_start_of_step_fields) { + const auto& fname = it.first; const auto& f = get_field_out(fname); - - auto& tend = it.second; - tend.deep_copy(f); + auto& f_beg = it.second; + f_beg.deep_copy(f); } stop_timer(m_timer_prefix + this->name() + "::compute_tendencies"); } } void AtmosphereProcess::compute_step_tendencies (const double dt) { + using namespace ShortFieldTagsNames; if (m_compute_proc_tendencies) { + m_atm_logger->debug("[" + this->name() + "] computing tendencies..."); start_timer(m_timer_prefix + this->name() + "::compute_tendencies"); for (auto it : m_proc_tendencies) { + // Note: f_beg is nonconst, so we can store step tendency in it const auto& tname = it.first; const auto& fname = m_tend_to_field.at(tname); const auto& f = get_field_out(fname); + auto& f_beg = m_start_of_step_fields.at(fname); + auto& tend = it.second; - auto& tend = it.second; - - tend.update(f,1/dt,-1/dt); + // Compute tend from this atm proc step, then sum into overall atm timestep tendency + f_beg.update(f,1,-1); + tend.update(f_beg,1,1); } stop_timer(m_timer_prefix + this->name() + "::compute_tendencies"); } diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.hpp b/components/eamxx/src/share/atm_process/atmosphere_process.hpp index c642ff357bfb..5f7214a96d14 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.hpp @@ -115,7 +115,7 @@ class AtmosphereProcess : public ekat::enable_shared_from_this> + get_precondition_checks () { + return m_precondition_checks; + } + std::list> + get_postcondition_checks() { + return m_postcondition_checks; + } + std::pair + get_column_conservation_check() { + return m_column_conservation_check; + } + void init_step_tendencies (); void compute_step_tendencies (const double dt); @@ -257,10 +271,11 @@ class AtmosphereProcess : public ekat::enable_shared_from_this + void add_field (const std::string& name, const std::string& grid_name, + const std::list& groups, const int ps = 1) + { add_field(FieldRequest(name,grid_name,groups,ps)); } + template + void add_field (const std::string& name, const std::string& grid_name, const int ps = 1) + { add_field(name,grid_name,{},ps);} + template void add_field (const std::string& name, const FieldLayout& layout, const ekat::units::Units& u, const std::string& grid_name, @@ -312,6 +337,23 @@ class AtmosphereProcess : public ekat::enable_shared_from_this& groups, const int ps) { add_field(FieldRequest(fid,groups,ps)); } + // Group requests + template + void add_group (const std::string& name, const std::string& grid, const int ps, const Bundling b, + const DerivationType t, const std::string& src_name, const std::string& src_grid, + const std::list& excl = {}) + { add_group(GroupRequest(name,grid,ps,b,t,src_name,src_grid,excl)); } + + template + void add_group (const std::string& name, const std::string& grid_name, + const Bundling b = Bundling::NotNeeded) + { add_group (GroupRequest(name,grid_name,b)); } + + template + void add_group (const std::string& name, const std::string& grid_name, + const int pack_size, const Bundling b = Bundling::NotNeeded) + { add_group (GroupRequest(name,grid_name,pack_size,b)); } + template void add_field (const FieldRequest& req) { @@ -333,23 +375,6 @@ class AtmosphereProcess : public ekat::enable_shared_from_this - void add_group (const std::string& name, const std::string& grid, const int ps, const Bundling b, - const DerivationType t, const std::string& src_name, const std::string& src_grid, - const std::list& excl = {}) - { add_group(GroupRequest(name,grid,ps,b,t,src_name,src_grid,excl)); } - - template - void add_group (const std::string& name, const std::string& grid_name, - const Bundling b = Bundling::NotNeeded) - { add_group (GroupRequest(name,grid_name,b)); } - - template - void add_group (const std::string& name, const std::string& grid_name, - const int pack_size, const Bundling b = Bundling::NotNeeded) - { add_group (GroupRequest(name,grid_name,pack_size,b)); } - template void add_group (const GroupRequest& req) { @@ -494,6 +519,7 @@ class AtmosphereProcess : public ekat::enable_shared_from_this m_tend_to_field; strmap_t m_proc_tendencies; + strmap_t m_start_of_step_fields; // These maps help to retrieve a field/group stored in the lists above. E.g., // auto ptr = m_field_in_pointers[field_name][grid_name]; @@ -546,6 +572,9 @@ class AtmosphereProcess : public ekat::enable_shared_from_thistype()==AtmosphereProcessType::Group); @@ -360,25 +365,18 @@ add_nodes (const group_type& atm_procs) } else { // Create a node for the process // Node& node = m_nodes[proc->name()]; + int id = m_nodes.size(); m_nodes.push_back(Node()); Node& node = m_nodes.back();; node.id = id; node.name = proc->name(); - m_unmet_deps[id].clear(); + m_unmet_deps[id].clear(); // Ensures an entry for this id is in the map // Input fields for (const auto& f : proc->get_fields_in()) { const auto& fid = f.get_header().get_identifier(); const int fid_id = add_fid(fid); node.required.insert(fid_id); - auto it = m_fid_to_last_provider.find(fid_id); - if (it==m_fid_to_last_provider.end()) { - m_unmet_deps[id].insert(fid_id); - } else { - // Establish parent-child relationship - Node& parent = m_nodes[it->second]; - parent.children.push_back(node.id); - } } // Output fields @@ -404,30 +402,6 @@ add_nodes (const group_type& atm_procs) const auto& gr_fid = group.m_bundle->get_header().get_identifier(); const int gr_fid_id = add_fid(gr_fid); node.gr_required.insert(gr_fid_id); - auto it = m_fid_to_last_provider.find(gr_fid_id); - if (it==m_fid_to_last_provider.end()) { - // It might still be ok, as long as there is a provider for all the fields in the group - bool all_members_have_providers = true; - for (auto it_f : group.m_fields) { - const auto& fid = it_f.second->get_header().get_identifier(); - const int fid_id = add_fid(fid); - auto it_p = m_fid_to_last_provider.find(fid_id); - if (it_p==m_fid_to_last_provider.end()) { - m_unmet_deps[id].insert(fid_id); - all_members_have_providers = false; - } else { - Node& parent = m_nodes[it_p->second]; - parent.children.push_back(node.id); - } - } - if (!all_members_have_providers) { - m_unmet_deps[id].insert(gr_fid_id); - } - } else { - // Establish parent-child relationship - Node& parent = m_nodes[it->second]; - parent.children.push_back(node.id); - } m_gr_fid_to_group.emplace(gr_fid,group); } } @@ -459,7 +433,76 @@ add_nodes (const group_type& atm_procs) } } } - ++id; + } + } +} + +void AtmProcDAG::add_edges () { + for (auto& node : m_nodes) { + // First individual input fields. Add this node as a children + // of any *previous* node that computes them. If none provides + // them, add to the unmet deps list + for (auto id : node.required) { + auto it = m_fid_to_last_provider.find(id); + // Note: check that last provider id is SMALLER than this node id + if (it!=m_fid_to_last_provider.end() and it->secondsecond; + m_nodes[parent_id].children.push_back(node.id); + } else { + m_unmet_deps[node.id].insert(id); + } + } + // Then process groups, looking at both the bundled field and individual fields. + // NOTE: we don't know if the group as a whole is the last to be updated + // OR if each group member is updated after the last "group-update". + // So get the id of the last node that updates each field and the group, + // and use the most recent one + for (auto id : node.gr_required) { + const auto& gr_fid = m_fids[id]; + const auto& group = m_gr_fid_to_group.at(gr_fid); + const int size = group.m_info->size(); + + int last_group_update_id = -1; + std::vector last_members_update_id(size,-1); + + // First check when the group as a whole was last updated + auto it = m_fid_to_last_provider.find(id); + // Note: check that last provider id is SMALLER than this node id + if (it!=m_fid_to_last_provider.end() and it->secondsecond; + } + // Then check when each group member was last updated + int i=0; + for (auto f_it : group.m_fields) { + const auto& fid = f_it.second->get_header().get_identifier(); + auto fid_id = std::find(m_fids.begin(),m_fids.end(),fid) - m_fids.begin(); + it = m_fid_to_last_provider.find(fid_id); + // Note: check that last provider id is SMALLER than this node id + if (it!=m_fid_to_last_provider.end() and it->secondsecond; + } + ++i; + } + + auto min = *std::min_element(last_members_update_id.begin(),last_members_update_id.end()); + if (min==-1 && last_group_update_id==-1) { + // Nobody computed the group as a whole and some member was not computed + m_unmet_deps[node.id].insert(id); + } else if (min>last_group_update_id) { + // All members are updated after the group + for (auto fid_id : last_members_update_id) { + m_nodes[fid_id].children.push_back(node.id); + } + } else { + // Add the group provider as a parent, but also the provider of each + // field which is updated after the group + m_nodes[id].children.push_back(node.id); + for (auto fid_id : last_members_update_id) { + if (fid_id>last_group_update_id) { + m_nodes[fid_id].children.push_back(node.id); + } + } + } } } } diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp b/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp index 2e048b353f4c..1a88381ecc8f 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp @@ -32,6 +32,8 @@ class AtmProcDAG { void add_nodes (const group_type& atm_procs); + void add_edges (); + // Add fid to list of fields in the dag, and return its position. // If already stored, simply return its position int add_fid (const FieldIdentifier& fid); diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_group.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_group.cpp index f7276aee70ee..9d5ff4889295 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_group.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_group.cpp @@ -14,12 +14,9 @@ AtmosphereProcessGroup:: AtmosphereProcessGroup (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { - - // Get the string representation of the group - auto group_list_str = m_params.get("atm_procs_list"); - auto group_plist = ekat::parse_nested_list(group_list_str); - - m_group_size = group_plist.get("Num Entries"); + // Get the list of procs in the group + auto group_list = m_params.get>("atm_procs_list"); + m_group_size = group_list.size(); EKAT_REQUIRE_MSG (m_group_size>0, "Error! Invalid group size.\n"); if (m_group_size>1) { @@ -48,7 +45,7 @@ AtmosphereProcessGroup (const ekat::Comm& comm, const ekat::ParameterList& param // so we can recursively create groups. Groups are an impl detail, // so we don't expect users to register the APG in the factory. apf.register_product("group",&create_atmosphere_process); - for (int i=0; i(ekat::strint("Type",i)); - std::string ap_name, ap_type; - if (type_i=="Value") { - // This is a "named" atm proc. - ap_name = group_plist.get(ekat::strint("Entry",i)); - } else { - // This is a group defined "on the fly". Get its string representation - ap_name = group_plist.sublist(ekat::strint("Entry",i)).get("String"); - // Due to XML limitations, in CIME runs we need to create a name for atm proc groups - // that are defined via nested list string, and we do it by replacing ',' with '_', - // '(' with 'group.', and ')' with '.' - auto pos = ap_name.find(","); - while (pos!=std::string::npos) { - ap_name[pos] = '_'; - pos = ap_name.find(","); - } - pos = ap_name.find("("); - while (pos!=std::string::npos) { - ap_name[pos] = '.'; - ap_name.insert(pos,"group"); - pos = ap_name.find("("); - } - pos = ap_name.find(")"); - while (pos!=std::string::npos) { - ap_name[pos] = '.'; - pos = ap_name.find(")"); - } - ap_type = "Group"; + EKAT_ERROR_MSG("Error! Parallel schedule type not yet implemented.\n"); } // Get the params of this atm proc auto& params_i = m_params.sublist(ap_name); // Get type (defaults to name) - ap_type = type_i=="List" ? "Group" - : params_i.get("Type",ap_name); + const auto& ap_type = params_i.get("Type",ap_name); // Set logger in this ap params params_i.set("Logger",this->m_atm_logger); @@ -149,6 +112,40 @@ AtmosphereProcessGroup (const ekat::Comm& comm, const ekat::ParameterList& param } } +std::shared_ptr +AtmosphereProcessGroup::get_process_nonconst (const std::string& name) const { + EKAT_REQUIRE_MSG(has_process(name), "Error! Process "+name+" requested from group "+this->name()+ + " but is not constained in this group.\n"); + + std::shared_ptr return_process; + for (auto& process: m_atm_processes) { + if (process->type() == AtmosphereProcessType::Group) { + const auto* group = dynamic_cast(process.get()); + return group->get_process_nonconst(name); + } else if (process->name() == name) { + return_process = process; + break; + } + } + return return_process; +} + +bool AtmosphereProcessGroup::has_process(const std::string& name) const { + for (auto& process: m_atm_processes) { + if (process->type() == AtmosphereProcessType::Group) { + const auto* group = dynamic_cast(process.get()); + if (group->has_process(name)) { + return true; + } + } else { + if (process->name() == name) { + return true; + } + } + } + return false; +} + void AtmosphereProcessGroup::set_grids (const std::shared_ptr grids_manager) { // The atm process group (APG) simply 'concatenates' required/computed @@ -288,7 +285,7 @@ setup_column_conservation_checks (const std::shared_ptrname() + "\". " "This check is column local and therefore can only be run " "on physics processes.\n"); - + // Query the computed fields for this atm process and see if either the mass or energy computation // might be changed after the process has run. If no field used in the mass or energy calculate // is updated by this process, there is no need to run the check. @@ -348,6 +345,25 @@ void AtmosphereProcessGroup::add_postcondition_nan_checks () const { } } +void AtmosphereProcessGroup::add_additional_data_fields_to_property_checks (const Field& data_field) { + for (auto proc : m_atm_processes) { + auto group = std::dynamic_pointer_cast(proc); + if (group) { + group->add_additional_data_fields_to_property_checks(data_field); + } else { + for (auto& prop_check : proc->get_precondition_checks()) { + prop_check.second->set_additional_data_field(data_field); + } + for (auto& prop_check : proc->get_postcondition_checks()) { + prop_check.second->set_additional_data_field(data_field); + } + if (proc->has_column_conservation_check()) { + proc->get_column_conservation_check().second->set_additional_data_field(data_field); + } + } + } +} + void AtmosphereProcessGroup::initialize_impl (const RunType run_type) { for (auto& atm_proc : m_atm_processes) { atm_proc->initialize(timestamp(),run_type); diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp b/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp index 2e33057f246d..a5c8f0a781d7 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp @@ -59,6 +59,13 @@ class AtmosphereProcessGroup : public AtmosphereProcess return m_atm_processes.at(i); } + // Returns atmosphere process if contained in this group, error out if not + std::shared_ptr get_process_nonconst (const std::string& name) const; + + // returns true if this group contains the process (either directly or within + // a nested group), false if not + bool has_process(const std::string& name) const; + ScheduleType get_schedule_type () const { return m_group_schedule_type; } // Computes total number of bytes needed for local variables @@ -95,6 +102,9 @@ class AtmosphereProcessGroup : public AtmosphereProcess // (that are on the same grid) at the location of the fail. void add_postcondition_nan_checks () const; + // Add additional data fields to all property checks in the group + void add_additional_data_fields_to_property_checks (const Field& data_field); + protected: // Adds fid to the list of required/computed fields of the group (as a whole). diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index d0bf8d676373..37cb251d7796 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -1,66 +1,16 @@ #include "share/atm_process/atmosphere_process.hpp" #include "share/field/field_utils.hpp" #include "share/util/scream_array_utils.hpp" +#include "share/util/scream_bfbhash.hpp" #include "ekat/ekat_assert.hpp" #include namespace scream { - namespace { -typedef std::uint64_t HashType; - -KOKKOS_INLINE_FUNCTION void hash (const HashType v, HashType& accum) { - constexpr auto first_bit = 1ULL << 63; - accum += ~first_bit & v; // no overflow - accum ^= first_bit & v; // handle most significant bit -} - -KOKKOS_INLINE_FUNCTION void hash (const double v_, HashType& accum) { - static_assert(sizeof(double) == sizeof(HashType), - "HashType must have size sizeof(double)."); - HashType v; - std::memcpy(&v, &v_, sizeof(HashType)); - hash(v, accum); -} - -// For Kokkos::parallel_reduce. -template -struct HashReducer { - typedef HashReducer reducer; - typedef HashType value_type; - typedef Kokkos::View result_view_type; - - KOKKOS_INLINE_FUNCTION HashReducer (value_type& value_) : value(value_) {} - KOKKOS_INLINE_FUNCTION void join (value_type& dest, const value_type& src) const { hash(src, dest); } - KOKKOS_INLINE_FUNCTION void init (value_type& val) const { val = 0; } - KOKKOS_INLINE_FUNCTION value_type& reference () const { return value; } - KOKKOS_INLINE_FUNCTION bool references_scalar () const { return true; } - KOKKOS_INLINE_FUNCTION result_view_type view () const { return result_view_type(&value, 1); } - -private: - value_type& value; -}; - -void reduce_hash (void* invec, void* inoutvec, int* len, MPI_Datatype* /* datatype */) { - const int n = *len; - const auto* s = reinterpret_cast(invec); - auto* d = reinterpret_cast(inoutvec); - for (int i = 0; i < n; ++i) hash(s[i], d[i]); -} - -int all_reduce_HashType (MPI_Comm comm, const HashType* sendbuf, HashType* rcvbuf, - int count) { - static_assert(sizeof(long long int) == sizeof(HashType), - "HashType must have size sizeof(long long int)."); - MPI_Op op; - MPI_Op_create(reduce_hash, true, &op); - const auto stat = MPI_Allreduce(sendbuf, rcvbuf, count, MPI_LONG_LONG_INT, op, comm); - MPI_Op_free(&op); - return stat; -} using ExeSpace = KokkosTypes::ExeSpace; +using bfbhash::HashType; void hash (const Field::view_dev_t& v, const FieldLayout& lo, HashType& accum_out) { @@ -68,10 +18,10 @@ void hash (const Field::view_dev_t& v, Kokkos::parallel_reduce( Kokkos::RangePolicy(0, lo.size()), KOKKOS_LAMBDA(const int idx, HashType& accum) { - hash(v(idx), accum); - }, HashReducer<>(accum)); + bfbhash::hash(v(idx), accum); + }, bfbhash::HashReducer<>(accum)); Kokkos::fence(); - hash(accum, accum_out); + bfbhash::hash(accum, accum_out); } void hash (const Field::view_dev_t& v, @@ -83,10 +33,10 @@ void hash (const Field::view_dev_t& v, KOKKOS_LAMBDA(const int idx, HashType& accum) { int i, j; unflatten_idx(idx, dims, i, j); - hash(v(i,j), accum); - }, HashReducer<>(accum)); + bfbhash::hash(v(i,j), accum); + }, bfbhash::HashReducer<>(accum)); Kokkos::fence(); - hash(accum, accum_out); + bfbhash::hash(accum, accum_out); } void hash (const Field::view_dev_t& v, @@ -98,10 +48,10 @@ void hash (const Field::view_dev_t& v, KOKKOS_LAMBDA(const int idx, HashType& accum) { int i, j, k; unflatten_idx(idx, dims, i, j, k); - hash(v(i,j,k), accum); - }, HashReducer<>(accum)); + bfbhash::hash(v(i,j,k), accum); + }, bfbhash::HashReducer<>(accum)); Kokkos::fence(); - hash(accum, accum_out); + bfbhash::hash(accum, accum_out); } void hash (const Field::view_dev_t& v, @@ -113,10 +63,10 @@ void hash (const Field::view_dev_t& v, KOKKOS_LAMBDA(const int idx, HashType& accum) { int i, j, k, m; unflatten_idx(idx, dims, i, j, k, m); - hash(v(i,j,k,m), accum); - }, HashReducer<>(accum)); + bfbhash::hash(v(i,j,k,m), accum); + }, bfbhash::HashReducer<>(accum)); Kokkos::fence(); - hash(accum, accum_out); + bfbhash::hash(accum, accum_out); } void hash (const Field::view_dev_t& v, @@ -128,36 +78,44 @@ void hash (const Field::view_dev_t& v, KOKKOS_LAMBDA(const int idx, HashType& accum) { int i, j, k, m, n; unflatten_idx(idx, dims, i, j, k, m, n); - hash(v(i,j,k,m,n), accum); - }, HashReducer<>(accum)); + bfbhash::hash(v(i,j,k,m,n), accum); + }, bfbhash::HashReducer<>(accum)); Kokkos::fence(); - hash(accum, accum_out); + bfbhash::hash(accum, accum_out); } -void hash (const std::list& fgs, HashType& accum_out) { - +void hash (const Field& f, HashType& accum) { + const auto& hd = f.get_header(); + const auto& id = hd.get_identifier(); + if (id.data_type() != DataType::DoubleType) return; + const auto& lo = id.get_layout(); + const auto rank = lo.rank(); + switch (rank) { + case 1: hash(f.get_view(), lo, accum); break; + case 2: hash(f.get_view(), lo, accum); break; + case 3: hash(f.get_view(), lo, accum); break; + case 4: hash(f.get_view(), lo, accum); break; + case 5: hash(f.get_view(), lo, accum); break; + default: break; + } } void hash (const std::list& fs, HashType& accum) { - for (const auto& f : fs) { - const auto& hd = f.get_header(); - const auto& id = hd.get_identifier(); - if (id.data_type() != DataType::DoubleType) continue; - const auto& lo = id.get_layout(); - const auto rank = lo.rank(); - switch (rank) { - case 1: hash(f.get_view(), lo, accum); break; - case 2: hash(f.get_view(), lo, accum); break; - case 3: hash(f.get_view(), lo, accum); break; - case 4: hash(f.get_view(), lo, accum); break; - case 5: hash(f.get_view(), lo, accum); break; - default: continue; - } - } + for (const auto& f : fs) + hash(f, accum); } + +void hash (const std::list& fgs, HashType& accum) { + for (const auto& g : fgs) + for (const auto& e : g.m_fields) + hash(*e.second, accum); +} + } // namespace anon -void AtmosphereProcess::print_global_state_hash (const std::string& label) const { +void AtmosphereProcess +::print_global_state_hash (const std::string& label, const bool in, const bool out, + const bool internal) const { static constexpr int nslot = 3; HashType laccum[nslot] = {0}; hash(m_fields_in, laccum[0]); @@ -166,19 +124,21 @@ void AtmosphereProcess::print_global_state_hash (const std::string& label) const hash(m_groups_out, laccum[1]); hash(m_internal_fields, laccum[2]); HashType gaccum[nslot]; - all_reduce_HashType(m_comm.mpi_comm(), laccum, gaccum, nslot); + bfbhash::all_reduce_HashType(m_comm.mpi_comm(), laccum, gaccum, nslot); + const bool show[] = {in, out, internal}; if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", - timestamp().get_year(), timestamp().frac_of_year_in_days(), - i, gaccum[i], label.c_str()); + if (show[i]) + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + timestamp().get_year(), timestamp().frac_of_year_in_days(), + i, gaccum[i], label.c_str()); } void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) const { HashType laccum = 0; hash(m_fields_in, laccum); HashType gaccum; - all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); + bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 154b8d0025ea..71a84fd6f7d9 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -150,13 +150,9 @@ void Field::allocate_view () // Short names const auto& id = m_header->get_identifier(); - const auto& layout = id.get_layout_ptr(); + const auto& layout = id.get_layout(); auto& alloc_prop = m_header->get_alloc_properties(); - // Check the identifier has all the dimensions set - EKAT_REQUIRE_MSG(layout->are_dimensions_set(), - "Error! Cannot allocate the view until all the field's dimensions are set.\n"); - // Commit the allocation properties alloc_prop.commit(layout); diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 9ce8eed940dd..b8445bce02bf 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -95,6 +95,10 @@ class Field { // Constructor(s) Field () = default; explicit Field (const identifier_type& id); + template::value>::type> + Field (const identifier_type& id, + const ViewT& view_d); Field (const Field& src) = default; ~Field () = default; @@ -126,7 +130,7 @@ class Field { // Like the method above, but only for rank-1 fields, returning a view with LayoutStride. // This is safer to use for fields that could be a subfield of another one, since a - // rank-1 view that is the subview of a 2d one along the 2nd index cannot have + // rank-1 view that is the subview of a 2d one along the 2nd index cannot have // LayoutRight, and must have LayoutStride instead. template get_strided_view_type @@ -175,12 +179,12 @@ class Field { // view.data ptr. template ST* get_internal_view_data_unsafe () const { - // Check that the scalar type is correct + // Check that the scalar type is correct using nonconst_ST = typename std::remove_const::type; EKAT_REQUIRE_MSG ((field_valid_data_types().at()==m_header->get_identifier().data_type() or std::is_same::value), "Error! Attempt to access raw field pointere with the wrong scalar type.\n"); - + return reinterpret_cast(get_view_impl().data()); } @@ -212,6 +216,10 @@ class Field { template void scale (const ST beta); + // Scale a field y as y=y*x where x is also a field + template + void scale (const Field& x); + // Returns a subview of this field, slicing at entry k along dimension idim // NOTES: // - the output field stores *the same* 1d view as this field. In order @@ -271,7 +279,7 @@ class Field { void deep_copy_impl (const Field& src); template - void update_impl (const Field& x, const ST alpha, const ST beta); + void update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val); protected: diff --git a/components/eamxx/src/share/field/field_alloc_prop.cpp b/components/eamxx/src/share/field/field_alloc_prop.cpp index afc694064447..7c589678defb 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.cpp +++ b/components/eamxx/src/share/field/field_alloc_prop.cpp @@ -3,7 +3,8 @@ namespace scream { FieldAllocProp::FieldAllocProp (const int scalar_size) - : m_value_type_sizes (1,scalar_size) + : m_layout (FieldLayout::invalid()) + , m_value_type_sizes (1,scalar_size) , m_scalar_type_size (scalar_size) , m_pack_size_max (1) , m_alloc_size (0) @@ -37,54 +38,41 @@ subview (const int idim, const int k, const bool dynamic) const { "Error! Subview requires alloc properties to be committed.\n"); EKAT_REQUIRE_MSG (idim==0 || idim==1, "Error! Subviewing is only allowed along first or second dimension.\n"); - EKAT_REQUIRE_MSG (idimrank(), + EKAT_REQUIRE_MSG (idim=0 && kdim(idim), + EKAT_REQUIRE_MSG (k>=0 && kdim(idim); - props.m_subview_info = SubviewInfo(idim,k,m_layout->dim(idim),dynamic); - - // The output props should still store a FieldLayout, in case - // they are further subviewed. We have all we need here to build - // a layout from scratch, but it would duplicate what is inside - // the FieldIdentifier that will be built for the corresponding - // field. Therefore, we'll require the user to still call 'commit', - // passing the layout from the field id. - // HOWEVER, this may cause bugs, cause the user might make a mistake, - // and pass the wrong layout. Therefore, we build a "temporary" one, - // which will be replaced during the call to 'commit'. - std::vector tags = m_layout->tags(); - std::vector dims = m_layout->dims(); - tags.erase(tags.begin()+idim); - dims.erase(dims.begin()+idim); - props.m_layout = std::make_shared(tags,dims); + props.m_layout = m_layout.strip_dim(idim); // Output is contioguous if either // - this->m_contiguous=true AND idim==0 - // - m_layout->dim(i)==1 for all irank()==0 - props.m_contiguous = m_contiguous || props.m_layout->rank()==0; - for (int i=0; idim(i)>0) { - props.m_contiguous = false; - break; - } - } + // - m_layout.dim(i)==1 for all irank()-1; + props.m_subview_info = SubviewInfo(idim,k,m_layout.dim(idim),dynamic); + + // Figure out strides/packs + const int rm1 = m_layout.rank()-1; if (idim==rm1) { - // We're slicing the possibly padded dim, so everything else is as in the layout - props.m_last_extent = m_layout->dim(idim); + // We're slicing the possibly padded dim, so everything else is as in the layout, + // and there is no packing + props.m_last_extent = m_layout.dim(idim); + props.m_pack_size_max = 1; + props.m_alloc_size = m_alloc_size / m_last_extent; } else { - // We are keeping the last dim, so same last extent + // We are keeping the last dim, so same last extent and max pack size props.m_last_extent = m_last_extent; + props.m_pack_size_max = m_pack_size_max; + props.m_alloc_size = m_alloc_size / m_layout.dim(idim); } return props; } @@ -113,7 +101,7 @@ void FieldAllocProp::request_allocation (const FieldAllocProp& src) int FieldAllocProp::get_padding () const { EKAT_REQUIRE_MSG(is_committed(), "Error! You cannot query the allocation padding until after calling commit()."); - int padding = m_last_extent - m_layout->dims().back(); + int padding = m_layout.rank()==0 ? 0 : m_last_extent - m_layout.dims().back(); return padding; } @@ -131,63 +119,55 @@ void FieldAllocProp::reset_subview_idx (const int idx) { m_subview_info.slice_idx = idx; } -void FieldAllocProp::commit (const layout_ptr_type& layout) +void FieldAllocProp::commit (const layout_type& layout) { if (is_committed()) { // TODO: should we issue a warning? Error? For now, I simply do nothing return; } - EKAT_REQUIRE_MSG (layout, - "Error! Invalid input layout pointer.\n"); - - if (m_alloc_size>0) { - // This obj was created as a subview of another alloc props obj. - // Check that input layout matches the stored one, then replace the ptr. - EKAT_REQUIRE_MSG (*layout==*m_layout, - "Error! The input field layout does not match the stored one.\n"); - - m_layout = layout; - m_committed = true; - return; - } - // Sanity checks: we must have requested at least one value type, and the identifier needs all dimensions set by now. EKAT_REQUIRE_MSG(m_value_type_sizes.size()>0, "Error! No value types requested for the allocation.\n"); - EKAT_REQUIRE_MSG(layout->are_dimensions_set(), + EKAT_REQUIRE_MSG(layout.are_dimensions_set(), "Error! You need all field dimensions set before committing the allocation properties.\n"); - // Store pointer to layout for future use (in case subview is called) + // Store layout for future use (in case subview is called) m_layout = layout; - // Loop on all value type sizes. - m_last_extent = 0; - int last_phys_extent = m_layout->dims().back(); - for (auto vts : m_value_type_sizes) { - // The number of scalar_type in a value_type - const int vt_len = vts / m_scalar_type_size; + if (m_layout.rank()==0) { + // Zero-dimensional fields are supported. In this case allocate a single + // scalar, but set the last extent to 0 since we have no dimension. + m_alloc_size = m_scalar_type_size; + m_pack_size_max = 1; + m_last_extent = 0; + } else { + // Loop on all value type sizes. + m_last_extent = 0; + int last_phys_extent = m_layout.dims().back(); + for (auto vts : m_value_type_sizes) { + // The number of scalar_type in a value_type + const int vt_len = vts / m_scalar_type_size; - // Update the max pack size - m_pack_size_max = std::max(m_pack_size_max,vt_len); + // Update the max pack size + m_pack_size_max = std::max(m_pack_size_max,vt_len); - // The number of value_type's needed in the fast-striding dimension - const int num_vt = (last_phys_extent + vt_len - 1) / vt_len; + // The number of value_type's needed in the fast-striding dimension + const int num_vt = (last_phys_extent + vt_len - 1) / vt_len; - // The total number of scalars with num_vt value_type entries - const int num_st = num_vt*vt_len; + // The total number of scalars with num_vt value_type entries + const int num_st = num_vt*vt_len; - // The size of such allocation (along the last dim only) - m_last_extent = std::max(m_last_extent, num_st); - } + // The size of such allocation (along the last dim only) + m_last_extent = std::max(m_last_extent, num_st); + } - // If we have a partitioned grid, with some ranks not owning any grid point, - // we may end up with a layout of size 0 - if (m_layout->size()==0) { - m_alloc_size = 0; - } else { - m_alloc_size = (m_layout->size() / last_phys_extent) // All except the last dimension - * m_last_extent * m_scalar_type_size; + if (m_layout.size()>0) { + m_alloc_size = m_layout.size() / last_phys_extent // All except the last dimension + * m_last_extent * m_scalar_type_size; // Last dimension must account for padding (if any) + } else { + m_alloc_size = 0; + } } m_contiguous = true; diff --git a/components/eamxx/src/share/field/field_alloc_prop.hpp b/components/eamxx/src/share/field/field_alloc_prop.hpp index 3183c5c0e3a0..52c5d095057f 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.hpp +++ b/components/eamxx/src/share/field/field_alloc_prop.hpp @@ -103,7 +103,7 @@ class FieldAllocProp { void request_allocation (const FieldAllocProp& src); // Locks the allocation properties, preventing furter value types requests - void commit (const layout_ptr_type& layout); + void commit (const layout_type& layout); // For dynamic subfield, reset the slice index void reset_subview_idx (const int idx); @@ -145,7 +145,7 @@ class FieldAllocProp { protected: // The FieldLayout associated to this allocation - layout_ptr_type m_layout; + layout_type m_layout; // The list of requested value types for this allocation std::vector m_value_type_sizes; diff --git a/components/eamxx/src/share/field/field_header.cpp b/components/eamxx/src/share/field/field_header.cpp index 15b09ad4f464..9bb642f2236f 100644 --- a/components/eamxx/src/share/field/field_header.cpp +++ b/components/eamxx/src/share/field/field_header.cpp @@ -8,9 +8,14 @@ namespace scream FieldHeader::FieldHeader (const identifier_type& id) : m_identifier (id) , m_tracking (create_tracking()) - , m_alloc_prop (get_type_size(id.data_type())) { - // Nothing to be done here + m_alloc_prop = std::make_shared(get_type_size(id.data_type())); + m_extra_data = std::make_shared(); + + // Let's add immediately this att, so that users don't need to check + // if it already exist before adding string attributes for io. + using stratts_t = std::map; + set_extra_data("io: string attributes",stratts_t()); } void FieldHeader:: @@ -19,12 +24,12 @@ set_extra_data (const std::string& key, const bool throw_if_existing) { if (throw_if_existing) { - EKAT_REQUIRE_MSG (m_extra_data.find(key)==m_extra_data.end(), + EKAT_REQUIRE_MSG (m_extra_data->find(key)==m_extra_data->end(), "Error! Key '" + key + "' already existing in " "the extra data map of field '" + m_identifier.get_id_string() + "'.\n"); - m_extra_data[key] = data; + (*m_extra_data)[key] = data; } else { - m_extra_data[key] = data; + (*m_extra_data)[key] = data; } } @@ -49,8 +54,6 @@ create_subfield_header (const FieldIdentifier& id, // Sanity checks EKAT_REQUIRE_MSG (parent!=nullptr, "Error! Invalid pointer for parent header.\n"); - EKAT_REQUIRE_MSG (id.get_layout_ptr()!=nullptr, - "Error! Input field identifier has an invalid layout pointer.\n"); // Create header, and set up parent/child auto fh = create_header(id); @@ -64,8 +67,7 @@ create_subfield_header (const FieldIdentifier& id, } // Create alloc props - fh->m_alloc_prop = parent->get_alloc_properties().subview(idim,k,dynamic); - fh->m_alloc_prop.commit(id.get_layout_ptr()); + fh->m_alloc_prop = std::make_shared(parent->get_alloc_properties().subview(idim,k,dynamic)); return fh; } diff --git a/components/eamxx/src/share/field/field_header.hpp b/components/eamxx/src/share/field/field_header.hpp index 46b0c02f86fa..511c6b6f5056 100644 --- a/components/eamxx/src/share/field/field_header.hpp +++ b/components/eamxx/src/share/field/field_header.hpp @@ -72,11 +72,15 @@ class FieldHeader : public FamilyTracking { const std::shared_ptr& get_tracking_ptr () const { return m_tracking; } // Get the allocation properties - const FieldAllocProp& get_alloc_properties () const { return m_alloc_prop; } - FieldAllocProp& get_alloc_properties () { return m_alloc_prop; } + const FieldAllocProp& get_alloc_properties () const { return *m_alloc_prop; } + FieldAllocProp& get_alloc_properties () { return *m_alloc_prop; } // Get the extra data - const extra_data_type& get_extra_data () const { return m_extra_data; } + template + const T& get_extra_data (const std::string& key) const; + template + T& get_extra_data (const std::string& key); + bool has_extra_data (const std::string& key) const; std::shared_ptr alias (const std::string& name) const; @@ -88,19 +92,67 @@ class FieldHeader : public FamilyTracking { std::shared_ptr, const int, const int, const bool); + // NOTE: the identifier *cannot* be a shared_ptr, b/c we + // don't foresee sharing an identifier between two + // field header instances. + // Static information about the field: name, rank, tags - identifier_type m_identifier; + identifier_type m_identifier; // Tracking of the field - std::shared_ptr m_tracking; + std::shared_ptr m_tracking; // Allocation properties - FieldAllocProp m_alloc_prop; + std::shared_ptr m_alloc_prop; // Extra data associated with this field - extra_data_type m_extra_data; + std::shared_ptr m_extra_data; }; +template +inline const T& FieldHeader:: +get_extra_data (const std::string& key) const +{ + EKAT_REQUIRE_MSG (has_extra_data(key), + "Error! Extra data not found in field header.\n" + " - field name: " + m_identifier.name() + "\n" + " - extra data: " + key + "\n"); + auto a = m_extra_data->at(key); + EKAT_REQUIRE_MSG ( a.isType(), + "Error! Attempting to access extra data using the wrong type.\n" + " - field name : " + m_identifier.name() + "\n" + " - extra data : " + key + "\n" + " - actual type : " + std::string(a.content().type().name()) + "\n" + " - requested type: " + std::string(typeid(T).name()) + ".\n"); + + return ekat::any_cast(a); +} + +template +inline T& FieldHeader:: +get_extra_data (const std::string& key) +{ + EKAT_REQUIRE_MSG (has_extra_data(key), + "Error! Extra data not found in field header.\n" + " - field name: " + m_identifier.name() + "\n" + " - extra data: " + key + "\n"); + auto a = m_extra_data->at(key); + EKAT_REQUIRE_MSG ( a.isType(), + "Error! Attempting to access extra data using the wrong type.\n" + " - field name : " + m_identifier.name() + "\n" + " - extra data : " + key + "\n" + " - actual type : " + std::string(a.content().type().name()) + "\n" + " - requested type: " + std::string(typeid(T).name()) + ".\n"); + + return ekat::any_cast(a); +} + +inline bool FieldHeader:: +has_extra_data (const std::string& key) const +{ + return m_extra_data->find(key)!=m_extra_data->end(); +} + // Use this free function to exploit features of enable_from_this template inline std::shared_ptr diff --git a/components/eamxx/src/share/field/field_identifier.cpp b/components/eamxx/src/share/field/field_identifier.cpp index fc730df0baba..56974aa06968 100644 --- a/components/eamxx/src/share/field/field_identifier.cpp +++ b/components/eamxx/src/share/field/field_identifier.cpp @@ -21,11 +21,12 @@ FieldIdentifier (const std::string& name, const std::string& grid_name, const DataType data_type) : m_name (name) + , m_layout (layout) , m_units (units) , m_grid_name (grid_name) , m_data_type (data_type) { - set_layout (layout); + update_identifier(); } FieldIdentifier @@ -37,38 +38,15 @@ alias (const std::string& name) const return fid; } -void FieldIdentifier::set_layout (const layout_type& layout) { - set_layout(std::make_shared(layout)); -} - -void FieldIdentifier::set_layout (const layout_ptr_type& layout) { - EKAT_REQUIRE_MSG (!m_layout, - "Error! You cannot reset the layout once it's set.\n"); - EKAT_REQUIRE_MSG (layout, - "Error! Invalid input layout pointer.\n"); - EKAT_REQUIRE_MSG (layout->are_dimensions_set(), - "Error! Input layout must have dimensions set.\n"); - - m_layout = layout; - update_identifier (); -} - void FieldIdentifier::update_identifier () { // Create a verbose identifier string. m_identifier = m_name + "[" + m_grid_name + "] <" + e2str(m_data_type); - if (m_layout->rank()>0) { - m_identifier += ":" + e2str(m_layout->tags()[0]); - for (int dim=1; dimrank(); ++dim) { - m_identifier += "," + e2str(m_layout->tags()[dim]); - } - m_identifier += ">(" + std::to_string(m_layout->dims()[0]); - for (int dim=1; dimrank(); ++dim) { - m_identifier += "," + std::to_string(m_layout->dims()[dim]); - } - m_identifier += ")"; - } else { - m_identifier += ">"; + auto print_tag = [](FieldTag t) {return e2str(t); }; + if (m_layout.rank()>0) { + m_identifier += ":" + ekat::join(m_layout.tags(),print_tag,","); } + m_identifier += ">"; + m_identifier += "(" + ekat::join(m_layout.dims(),",") + ")"; m_identifier += " [" + m_units.get_string() + "]"; } diff --git a/components/eamxx/src/share/field/field_identifier.hpp b/components/eamxx/src/share/field/field_identifier.hpp index e51ba6ac2fc1..7a2ff5c61337 100644 --- a/components/eamxx/src/share/field/field_identifier.hpp +++ b/components/eamxx/src/share/field/field_identifier.hpp @@ -40,7 +40,6 @@ field_valid_data_types () class FieldIdentifier { public: using layout_type = FieldLayout; - using layout_ptr_type = std::shared_ptr; using ci_string = ekat::CaseInsensitiveString; using Units = ekat::units::Units; @@ -66,8 +65,7 @@ class FieldIdentifier { // Name and layout informations const std::string& name () const { return m_name; } - const layout_type& get_layout () const { return *m_layout; } - const layout_ptr_type& get_layout_ptr () const { return m_layout; } + const layout_type& get_layout () const { return m_layout; } const Units& get_units () const { return m_units; } const std::string& get_grid_name () const { return m_grid_name; } DataType data_type () const { return m_data_type; } @@ -80,10 +78,6 @@ class FieldIdentifier { // ----- Setters ----- // - // Note: as soon as the layout is set, it cannot be changed. - void set_layout (const layout_type& layout); - void set_layout (const layout_ptr_type& layout); - // We reimplement the equality operator for identifiers comparison (needed for some std container) friend bool operator== (const FieldIdentifier&, const FieldIdentifier&); friend bool operator< (const FieldIdentifier&, const FieldIdentifier&); @@ -94,13 +88,13 @@ class FieldIdentifier { ci_string m_name; - layout_ptr_type m_layout; + layout_type m_layout; Units m_units; ci_string m_grid_name; - DataType m_data_type; + DataType m_data_type; // The identifier string is a conveniet way to display the information of // the identifier, so that it can be easily read. diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 2c3b6f20f319..0efe46771422 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -3,10 +3,87 @@ #include "share/field/field.hpp" #include "share/util/scream_array_utils.hpp" +#include "share/util/scream_universal_constants.hpp" + +#include namespace scream { +template +Field:: +Field (const identifier_type& id, + const ViewT& view_d) + : Field(id) +{ + constexpr auto N = ViewT::Rank; + using ScalarT = typename ViewT::traits::value_type; + using ExeSpace = typename ViewT::traits::execution_space; + + EKAT_REQUIRE_MSG ( (std::is_same::value), + "Error! This constructor of Field requires a view from device.\n"); + + EKAT_REQUIRE_MSG (id.data_type()==get_data_type(), + "Error! Input view data type does not match what is stored in the field identifier.\n" + " - field name: " + id.name() + "\n" + " - field data type: " + e2str(id.data_type()) + "\n"); + + const auto& fl = id.get_layout(); + EKAT_REQUIRE_MSG (N==fl.rank(), + "Error! This constructor of Field requires a device view of the correct rank.\n" + " - field name: " + id.name() + "\n" + " - field rank: " + std::to_string(fl.rank()) + "\n" + " - view rank : " + std::to_string(N) + "\n"); + for (int i=0; i<(N-1); ++i) { + EKAT_REQUIRE_MSG (view_d.extent_int(i)==fl.dims()[i], + "Error! Input view has the wrong i-th extent.\n" + " - field name: " + id.name() + "\n" + " - idim: " + std::to_string(i) + "\n" + " - layout i-th dim: " + std::to_string(fl.dims()[i]) + "\n" + " - view i-th dim: " + std::to_string(view_d.extent(i)) + "\n"); + } + + auto& alloc_prop = m_header->get_alloc_properties(); + if (N>0 and view_d.extent_int(N-1)!=fl.dims().back()) { + EKAT_REQUIRE_MSG (view_d.extent_int(N-1)>=fl.dims()[N-1], + "Error! Input view has the wrong last extent.\n" + " - field name: " + id.name() + "\n" + " - layout last dim: " + std::to_string(fl.dims()[N-1]) + "\n" + " - view last dim: " + std::to_string(view_d.extent(N-1)) + "\n"); + + // We have a padded view. We don't know what the pack size was, so we pick the largest + // power of 2 that divides the last extent + auto last_view_dim = view_d.extent_int(N-1); + int last_fl_dim = fl.dims().back(); + + // This should get the smallest pow of 2 that gives npacks*pack_size==view_last_dim + int ps = 1; + int packed_length = 0; + do { + ps *= 2; + auto npacks = (last_fl_dim + ps - 1) / ps; + packed_length = ps*npacks; + } + while (packed_length!=last_view_dim); + + alloc_prop.request_allocation(ps); + } + alloc_prop.commit(fl); + + // Create an unmanaged dev view, and its host mirror + const auto view_dim = alloc_prop.get_alloc_size(); + char* data = reinterpret_cast(view_d.data()); + std::cout << "fl: " << to_string(fl) << "\n" + << "view dim: " << view_dim << "\n"; + m_data.d_view = decltype(m_data.d_view)(data,view_dim); + m_data.h_view = Kokkos::create_mirror_view(m_data.d_view); + + // Since we created m_data.d_view from a raw pointer, we don't get any + // ref counting from the kokkos view. Hence, to ensure that the input view + // survives as long as this Field, we store it as extra data in the header + m_header->set_extra_data("orig_view",view_d); +} + template auto Field::get_view () const -> get_view_type @@ -193,15 +270,26 @@ deep_copy_impl (const Field& src) { const auto& layout = get_header().get_identifier().get_layout(); const auto& layout_src = src.get_header().get_identifier().get_layout(); EKAT_REQUIRE_MSG(layout==layout_src, - "ERROR: Unable to copy field " + src.get_header().get_identifier().name() + + "ERROR: Unable to copy field " + src.get_header().get_identifier().name() + " to field " + get_header().get_identifier().name() + ". Layouts don't match."); const auto rank = layout.rank(); + + // For rank 0 view, we only need to copy a single value and return + if (rank == 0) { + auto v = get_view< ST,HD>(); + auto v_src = src.get_view(); + v() = v_src(); + return; + } + // Note: we can't just do a deep copy on get_view_impl(), since this // field might be a subfield of another. We need the reshaped view. // Also, don't call Kokkos::deep_copy if this field and src have // different pack sizes. - auto src_alloc = src.get_header().get_alloc_properties().get_alloc_size(); - auto tgt_alloc = get_header().get_alloc_properties().get_alloc_size(); + auto src_alloc_props = src.get_header().get_alloc_properties(); + auto tgt_alloc_props = get_header().get_alloc_properties(); + auto src_alloc_size = src_alloc_props.get_alloc_size(); + auto tgt_alloc_size = tgt_alloc_props.get_alloc_size(); // If a manual parallel_for is required (b/c of alloc sizes difference), // we need to create extents (rather than just using the one in layout), @@ -217,14 +305,26 @@ deep_copy_impl (const Field& src) { switch (rank) { case 1: { - auto v = get_view< ST*,HD>(); - auto v_src = src.get_view(); - if (src_alloc==tgt_alloc) { - Kokkos::deep_copy(v,v_src); + if (src_alloc_props.contiguous() and tgt_alloc_props.contiguous()) { + auto v = get_view< ST*,HD>(); + auto v_src = src.get_view(); + if (src_alloc_size==tgt_alloc_size) { + Kokkos::deep_copy(v,v_src); + } else { + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + v(idx) = v_src(idx); + }); + } } else { - Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { - v(idx) = v_src(idx); - }); + auto v = get_strided_view< ST*,HD>(); + auto v_src = src.get_strided_view(); + if (src_alloc_size==tgt_alloc_size) { + Kokkos::deep_copy(v,v_src); + } else { + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + v(idx) = v_src(idx); + }); + } } } break; @@ -232,7 +332,7 @@ deep_copy_impl (const Field& src) { { auto v = get_view< ST**,HD>(); auto v_src = src.get_view(); - if (src_alloc==tgt_alloc) { + if (src_alloc_size==tgt_alloc_size) { Kokkos::deep_copy(v,v_src); } else { Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { @@ -247,7 +347,7 @@ deep_copy_impl (const Field& src) { { auto v = get_view< ST***,HD>(); auto v_src = src.get_view(); - if (src_alloc==tgt_alloc) { + if (src_alloc_size==tgt_alloc_size) { Kokkos::deep_copy(v,v_src); } else { Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { @@ -262,7 +362,7 @@ deep_copy_impl (const Field& src) { { auto v = get_view< ST****,HD>(); auto v_src = src.get_view(); - if (src_alloc==tgt_alloc) { + if (src_alloc_size==tgt_alloc_size) { Kokkos::deep_copy(v,v_src); } else { Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { @@ -277,7 +377,7 @@ deep_copy_impl (const Field& src) { { auto v = get_view< ST*****,HD>(); auto v_src = src.get_view(); - if (src_alloc==tgt_alloc) { + if (src_alloc_size==tgt_alloc_size) { Kokkos::deep_copy(v,v_src); } else { Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { @@ -303,10 +403,21 @@ void Field::deep_copy_impl (const ST value) { const auto& layout = get_header().get_identifier().get_layout(); const auto rank = layout.rank(); switch (rank) { + case 0: + { + auto v = get_view(); + v() = value; + } + break; case 1: { - auto v = get_view(); - Kokkos::deep_copy(v,value); + if (m_header->get_alloc_properties().contiguous()) { + auto v = get_view(); + Kokkos::deep_copy(v,value); + } else { + auto v = get_strided_view(); + Kokkos::deep_copy(v,value); + } } break; case 2: @@ -350,6 +461,17 @@ update (const Field& x, const ST alpha, const ST beta) { const auto& dt = data_type(); + // Determine if there is a FillValue that requires extra treatment. + ST fill_val = constants::DefaultFillValue().value; + + if (x.get_header().has_extra_data("mask_value")) { + if (typeid(ST) == typeid(int)) { + fill_val = x.get_header().get_extra_data("mask_value"); + } else { + fill_val = x.get_header().get_extra_data("mask_value"); + } + } + // If user passes, say, double alpha/beta for an int field, we should error out, warning about // a potential narrowing rounding. The other way around, otoh, is allowed (even though // there's an upper limit to the int values that a double can store, it is unlikely the user @@ -361,11 +483,11 @@ update (const Field& x, const ST alpha, const ST beta) " - coeff data type: " + e2str(dt_st) + "\n"); if (dt==DataType::IntType) { - return update_impl(x,alpha,beta); + return update_impl(x,alpha,beta,fill_val); } else if (dt==DataType::FloatType) { - return update_impl(x,alpha,beta); + return update_impl(x,alpha,beta,fill_val); } else if (dt==DataType::DoubleType) { - return update_impl(x,alpha,beta); + return update_impl(x,alpha,beta,fill_val); } else { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported field data type in Field::update.\n"); } @@ -377,6 +499,12 @@ scale (const ST beta) { const auto& dt = data_type(); + // Determine if there is a FillValue that requires extra treatment. + ST fill_val = constants::DefaultFillValue().value; + if (get_header().has_extra_data("mask_value")) { + fill_val = get_header().get_extra_data("mask_value"); + } + // If user passes, say, double beta for an int field, we should error out, warning about // a potential narrowing rounding. The other way around, otoh, is allowed (even though // there's an upper limit to the int values that a double can store, it is unlikely the user @@ -388,19 +516,49 @@ scale (const ST beta) " - coeff data type: " + e2str(dt_st) + "\n"); if (dt==DataType::IntType) { - return update_impl(*this,ST(0),beta); + return update_impl(*this,ST(0),beta,fill_val); } else if (dt==DataType::FloatType) { - return update_impl(*this,ST(0),beta); + return update_impl(*this,ST(0),beta,fill_val); } else if (dt==DataType::DoubleType) { - return update_impl(*this,ST(0),beta); + return update_impl(*this,ST(0),beta,fill_val); } else { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported field data type in Field::scale.\n"); } } +template +void Field:: +scale (const Field& x) +{ + const auto& dt = data_type(); + if (dt==DataType::IntType) { + int fill_val = constants::DefaultFillValue().value; + if (get_header().has_extra_data("mask_value")) { + fill_val = get_header().get_extra_data("mask_value"); + } + return update_impl(x,0,0,fill_val); + } else if (dt==DataType::FloatType) { + float fill_val = constants::DefaultFillValue().value; + if (get_header().has_extra_data("mask_value")) { + fill_val = get_header().get_extra_data("mask_value"); + } + return update_impl(x,0,0,fill_val); + } else if (dt==DataType::DoubleType) { + double fill_val = constants::DefaultFillValue().value; + if (get_header().has_extra_data("mask_value")) { + fill_val = get_header().get_extra_data("mask_value"); + } + return update_impl(x,0,0,fill_val); + } else { + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported field data type in Field::scale.\n"); + } +} + + + template void Field:: -update_impl (const Field& x, const ST alpha, const ST beta) +update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) { // Check x/y are allocated EKAT_REQUIRE_MSG (is_allocated(), @@ -450,15 +608,34 @@ update_impl (const Field& x, const ST alpha, const ST beta) auto policy = RangePolicy(0,x_l.size()); switch (x_l.rank()) { - case 1: + case 0: { - auto xv = x.get_view(); - auto yv = get_view< ST*,HD>(); - Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { - combine(xv(idx),yv(idx),alpha,beta); + auto xv = x.get_view(); + auto yv = get_view< ST,HD>(); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const int /*idx*/) { + combine_and_fill(xv(),yv(),fill_val,alpha,beta); }); } break; + case 1: + { + // Must handle the case where one of the two views is strided + if (x.get_header().get_alloc_properties().contiguous() and + get_header().get_alloc_properties().contiguous()) { + auto xv = x.get_view(); + auto yv = get_view< ST*,HD>(); + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + combine_and_fill(xv(idx),yv(idx),fill_val,alpha,beta); + }); + } else { + auto xv = x.get_strided_view(); + auto yv = get_strided_view< ST*,HD>(); + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + combine_and_fill(xv(idx),yv(idx),fill_val,alpha,beta); + }); + } + } + break; case 2: { auto xv = x.get_view(); @@ -466,7 +643,7 @@ update_impl (const Field& x, const ST alpha, const ST beta) Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j; unflatten_idx(idx,ext,i,j); - combine(xv(i,j),yv(i,j),alpha,beta); + combine_and_fill(xv(i,j),yv(i,j),fill_val,alpha,beta); }); } break; @@ -477,7 +654,7 @@ update_impl (const Field& x, const ST alpha, const ST beta) Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j,k; unflatten_idx(idx,ext,i,j,k); - combine(xv(i,j,k),yv(i,j,k),alpha,beta); + combine_and_fill(xv(i,j,k),yv(i,j,k),fill_val,alpha,beta); }); } break; @@ -488,7 +665,7 @@ update_impl (const Field& x, const ST alpha, const ST beta) Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j,k,l; unflatten_idx(idx,ext,i,j,k,l); - combine(xv(i,j,k,l),yv(i,j,k,l),alpha,beta); + combine_and_fill(xv(i,j,k,l),yv(i,j,k,l),fill_val,alpha,beta); }); } break; @@ -499,7 +676,7 @@ update_impl (const Field& x, const ST alpha, const ST beta) Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j,k,l,m; unflatten_idx(idx,ext,i,j,k,l,m); - combine(xv(i,j,k,l,m),yv(i,j,k,l,m),alpha,beta); + combine_and_fill(xv(i,j,k,l,m),yv(i,j,k,l,m),fill_val,alpha,beta); }); } break; @@ -510,7 +687,7 @@ update_impl (const Field& x, const ST alpha, const ST beta) Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j,k,l,m,n; unflatten_idx(idx,ext,i,j,k,l,m,n); - combine(xv(i,j,k,l,m,n),yv(i,j,k,l,m,n),alpha,beta); + combine_and_fill(xv(i,j,k,l,m,n),yv(i,j,k,l,m,n),fill_val,alpha,beta); }); } break; diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index d142707429fa..ba06fab8fb94 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -5,24 +5,6 @@ namespace scream { -FieldLayout::FieldLayout (const std::initializer_list& tags) - : m_rank(tags.size()) - , m_tags(tags) -{ - m_dims.resize(m_rank,-1); - m_extents = decltype(m_extents)("",m_rank); - Kokkos::deep_copy(m_extents,-1); -} - -FieldLayout::FieldLayout (const std::vector& tags) - : m_rank(tags.size()) - , m_tags(tags) -{ - m_dims.resize(m_rank,-1); - m_extents = decltype(m_extents)("",m_rank); - Kokkos::deep_copy(m_extents,-1); -} - FieldLayout::FieldLayout (const std::vector& tags, const std::vector& dims) : m_rank(tags.size()) @@ -30,8 +12,9 @@ FieldLayout::FieldLayout (const std::vector& tags, { m_dims.resize(m_rank,-1); m_extents = decltype(m_extents)("",m_rank); - Kokkos::deep_copy(m_extents,-1); - set_dimensions(dims); + for (int idim=0; idim vec_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; + auto it = std::find_first_of (m_tags.cbegin(),m_tags.cend(),vec_tags.cbegin(),vec_tags.cend()); + + EKAT_REQUIRE_MSG (it!=m_tags.cend(), + "Error! Could not find a vector tag in the layout.\n" + " - layout: " + to_string(*this) + "\n"); + + return std::distance(m_tags.cbegin(),it); +} + +FieldTag FieldLayout::get_vector_tag () const { + return m_tags[get_vector_dim()]; } FieldLayout FieldLayout::strip_dim (const FieldTag tag) const { @@ -69,6 +57,10 @@ FieldLayout FieldLayout::strip_dim (const FieldTag tag) const { } FieldLayout FieldLayout::strip_dim (const int idim) const { + EKAT_REQUIRE_MSG (idim>=0 and idim t = tags(); std::vector d = dims(); t.erase(t.begin()+idim); @@ -79,22 +71,12 @@ FieldLayout FieldLayout::strip_dim (const int idim) const { void FieldLayout::set_dimension (const int idim, const int dimension) { EKAT_REQUIRE_MSG(idim>=0 && idim=0, "Error! Dimensions must be non-negative."); - EKAT_REQUIRE_MSG(m_dims[idim] == -1, "Error! You cannot reset field dimensions once set.\n"); m_dims[idim] = dimension; - auto extents = Kokkos::create_mirror_view(m_extents); - Kokkos::deep_copy(extents,m_extents); - extents(idim) = dimension; - Kokkos::deep_copy(m_extents,extents); -} - -void FieldLayout::set_dimensions (const std::vector& dims) { - // Check, then set dims - EKAT_REQUIRE_MSG(dims.size()==static_cast(m_rank), - "Error! Input dimensions vector not properly sized."); - for (int idim=0; idim& field_tags) { @@ -107,6 +89,8 @@ LayoutType get_layout_type (const std::vector& field_tags) { const int n_element = count(tags,EL); const int n_column = count(tags,COL); const int ngp = count(tags,GP); + const int nvlevs = count(tags,LEV) + count(tags,ILEV); + const int ncomps = count(tags,CMP); // Start from undefined/invalid LayoutType result = LayoutType::Invalid; @@ -123,6 +107,14 @@ LayoutType get_layout_type (const std::vector& field_tags) { // Remove the column tag erase(tags,COL); + } else if (tags.size()==0) { + return LayoutType::Scalar0D; + } else if (tags.size()==1 and tags[0]==CMP) { + return LayoutType::Vector0D; + } else if (tags.size()==1 and nvlevs==1) { + return LayoutType::Scalar1D; + } else if (tags.size()==2 and ncomps==1 and nvlevs==1) { + return LayoutType::Vector1D; } else { // Not a supported layout. return result; @@ -130,15 +122,23 @@ LayoutType get_layout_type (const std::vector& field_tags) { // Get the size of what's left const auto size = tags.size(); + auto is_lev_tag = [](const FieldTag t) { + std::vector lev_tags = {LEV,ILEV}; + return ekat::contains(lev_tags,t); + }; + auto is_vec_tag = [](const FieldTag t) { + std::vector vec_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; + return ekat::contains(vec_tags,t); + }; switch (size) { case 0: result = LayoutType::Scalar2D; break; case 1: - // The only tag left should be 'CMP', 'TL', or 'LEV'/'ILEV' - if (tags[0]==CMP || tags[0]==TL) { + // The only tag left should be a vec tag, 'TL', or a lev tag + if (is_vec_tag(tags[0]) or tags[0]==TL) { result = LayoutType::Vector2D; - } else if (tags[0]==LEV || tags[0]==ILEV) { + } else if (is_lev_tag(tags[0])) { result = LayoutType::Scalar3D; } break; @@ -146,17 +146,16 @@ LayoutType get_layout_type (const std::vector& field_tags) { // Possible supported scenarios: // 1) // 2) - if ( (tags[1]==LEV || tags[1]==ILEV) && (tags[0]==CMP || tags[0]==TL)) { + if ( (is_vec_tag(tags[0]) or tags[0]==TL) and is_lev_tag(tags[1]) ) { result = LayoutType::Vector3D; - } else if (tags[0]==TL && tags[1]==CMP ) { + } else if (tags[0]==TL && is_vec_tag(tags[1]) ) { result = LayoutType::Tensor2D; } break; case 3: // The only supported scenario is: // 1) - if ( tags[0]==TL && tags[1]==CMP && - (tags[2]==LEV || tags[2]==ILEV)) { + if ( tags[0]==TL && is_vec_tag(tags[1]) && is_lev_tag(tags[2]) ) { result = LayoutType::Tensor3D; } } diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index 6fc5aec208d6..d0ca9b76b565 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -16,6 +16,10 @@ namespace scream // The type of the layout, that is, the kind of field it represent. enum class LayoutType { Invalid, + Scalar0D, + Vector0D, + Scalar1D, + Vector1D, Scalar2D, Vector2D, Tensor2D, @@ -27,6 +31,10 @@ enum class LayoutType { inline std::string e2str (const LayoutType lt) { std::string name; switch (lt) { + case LayoutType::Scalar0D: name = "Scalar0D"; break; + case LayoutType::Vector0D: name = "Vector0D"; break; + case LayoutType::Scalar1D: name = "Scalar1D"; break; + case LayoutType::Vector1D: name = "Vector1D"; break; case LayoutType::Scalar2D: name = "Scalar2D"; break; case LayoutType::Vector2D: name = "Vector2D"; break; case LayoutType::Tensor2D: name = "Tensor2D"; break; @@ -55,8 +63,6 @@ class FieldLayout { // Constructor(s) FieldLayout () = delete; FieldLayout (const FieldLayout&) = default; - FieldLayout (const std::initializer_list& tags); - FieldLayout (const std::vector& tags); FieldLayout (const std::vector& tags, const std::vector& dims); @@ -64,7 +70,7 @@ class FieldLayout { FieldLayout& operator= (const FieldLayout&) = default; // Create invalid layout - static FieldLayout invalid () { return FieldLayout({}); } + static FieldLayout invalid () { return FieldLayout({FieldTag::Invalid},{0}); } // ----- Getters ----- // @@ -84,7 +90,6 @@ class FieldLayout { long long size () const; - bool is_dimension_set (const int idim) const; bool are_dimensions_set () const; // Check if this layout is that of a vector field @@ -93,15 +98,14 @@ class FieldLayout { // If this is the layout of a vector field, get the idx of the vector dimension // Note: throws if is_vector_layout()==false. int get_vector_dim () const; + FieldTag get_vector_tag () const; FieldLayout strip_dim (const FieldTag tag) const; FieldLayout strip_dim (const int idim) const; // ----- Setters ----- // - // Note: as soon as a dimension is set, it cannot be changed. void set_dimension (const int idim, const int dimension); - void set_dimensions (const std::vector& dims); protected: @@ -137,8 +141,9 @@ inline int FieldLayout::dim (const int idim) const { } inline long long FieldLayout::size () const { - ekat::error::runtime_check(are_dimensions_set(), "Error! Field dimensions not yet set.\n",-1); - long long prod = m_rank>0 ? 1 : 0; + EKAT_REQUIRE_MSG(are_dimensions_set(), + "Error! Field dimensions not yet set.\n"); + long long prod = 1; for (int idim=0; idim& tags) const { return b; } -inline bool FieldLayout::is_dimension_set (const int idim) const { - ekat::error::runtime_check(idim>=0 && idim=0; -} - inline bool FieldLayout::are_dimensions_set () const { for (int idim=0; idimname() + "\n"); // Get or create the new field - if (!has_field(id.name())) { - EKAT_REQUIRE_MSG (id.data_type()==field_valid_data_types().at(), - "Error! While refactoring, we only allow the Field data type to be Real.\n" - " If you're done with refactoring, go back and fix things.\n"); - m_fields[id.name()] = std::make_shared(id); + if (req.incomplete) { + m_incomplete_requests.emplace_back(id.name(),id.get_grid_name()); } else { - // Make sure the input field has the same layout and units as the field already stored. - // TODO: this is the easiest way to ensure everyone uses the same units. - // However, in the future, we *may* allow different units, providing - // the users with conversion routines perhaps. - const auto id0 = m_fields[id.name()]->get_header().get_identifier(); - EKAT_REQUIRE_MSG(id.get_units()==id0.get_units(), - "Error! Field '" + id.name() + "' already registered with different units:\n" - " - input field units: " + to_string(id.get_units()) + "\n" - " - stored field units: " + to_string(id0.get_units()) + "\n" - " Please, check and make sure all atmosphere processes use the same units.\n"); - - EKAT_REQUIRE_MSG(id.get_layout()==id0.get_layout(), - "Error! Field '" + id.name() + "' already registered with different layout:\n" - " - input id: " + id.get_id_string() + "\n" - " - stored id: " + id0.get_id_string() + "\n" - " Please, check and make sure all atmosphere processes use the same layout for a given field.\n"); + if (!has_field(id.name())) { + + EKAT_REQUIRE_MSG (id.data_type()==field_valid_data_types().at(), + "Error! While refactoring, we only allow the Field data type to be Real.\n" + " If you're done with refactoring, go back and fix things.\n"); + m_fields[id.name()] = std::make_shared(id); + } else { + // Make sure the input field has the same layout and units as the field already stored. + // TODO: this is the easiest way to ensure everyone uses the same units. + // However, in the future, we *may* allow different units, providing + // the users with conversion routines perhaps. + const auto id0 = m_fields[id.name()]->get_header().get_identifier(); + EKAT_REQUIRE_MSG(id.get_units()==id0.get_units(), + "Error! Field '" + id.name() + "' already registered with different units:\n" + " - input field units: " + to_string(id.get_units()) + "\n" + " - stored field units: " + to_string(id0.get_units()) + "\n" + " Please, check and make sure all atmosphere processes use the same units.\n"); + + EKAT_REQUIRE_MSG(id.get_layout()==id0.get_layout(), + "Error! Field '" + id.name() + "' already registered with different layout:\n" + " - input id: " + id.get_id_string() + "\n" + " - stored id: " + id0.get_id_string() + "\n" + " Please, check and make sure all atmosphere processes use the same layout for a given field.\n"); + } } if (req.subview_info.dim_idx>=0) { // This is a request for a subfield. Store request info, so we can correctly set up // the subfield at the end of registration_ends() call m_subfield_requests.emplace(id.name(),req); - } else { + } else if (not req.incomplete) { // Make sure the field can accommodate the requested value type m_fields[id.name()]->get_header().get_alloc_properties().request_allocation(req.pack_size); } @@ -124,21 +129,23 @@ add_to_group (const std::string& field_name, const std::string& group_name) ft.add_to_group(group); } -bool FieldManager::has_field (const identifier_type& id) const -{ - return has_field(id.name()) && m_fields.at(id.name())->get_header().get_identifier()==id; +const FieldIdentifier& FieldManager::get_field_id (const std::string& name) const { + auto ptr = get_field_ptr(name); + EKAT_REQUIRE_MSG(ptr!=nullptr, + "Error! Field '" + name + "' not found.\n"); + return ptr->get_header().get_identifier(); } -Field FieldManager::get_field (const identifier_type& id) const { +Field FieldManager::get_field (const std::string& name) const { + EKAT_REQUIRE_MSG(m_repo_state==RepoState::Closed, "Error! Cannot get fields from the repo while registration has not yet completed.\n"); - auto ptr = get_field_ptr(id); - EKAT_REQUIRE_MSG(ptr!=nullptr, - "Error! Field identifier '" + id.get_id_string() + "' not found.\n"); + auto ptr = get_field_ptr(name); + EKAT_REQUIRE_MSG(ptr!=nullptr, "Error! Field " + name + " not found.\n"); return *ptr; } -Field FieldManager::get_field (const std::string& name) const { +Field& FieldManager::get_field (const std::string& name) { EKAT_REQUIRE_MSG(m_repo_state==RepoState::Closed, "Error! Cannot get fields from the repo while registration has not yet completed.\n"); @@ -224,6 +231,17 @@ void FieldManager::registration_begins () void FieldManager::registration_ends () { + // Before doing anything, ensure that for each incomplete requests the field has + // been registered with a complete FID. + for (const auto& it : m_incomplete_requests) { + EKAT_REQUIRE_MSG (has_field(it.first), + "Error! Found an incomplete FieldRequest for a field not registered with a valid identifier.\n" + " - field name: " + it.first + "\n" + " - grid name: " + it.second + "\n"); + } + // We no longer need this + m_incomplete_requests.clear(); + // This method is responsible of allocating the fields in the repo. The most delicate part is // the allocation of fields group, in the case where bundling is requested. In particular, // we want to try to honor as many requests for bundling as possible. If we can't accommodate @@ -282,7 +300,6 @@ void FieldManager::registration_ends () // to remove. // - // Start by processing group request. This function will ensure that, if there's a // request for group A that depends on the content of group B, the FieldGroupInfo // for group A is updated to contain the correct fields, based on the content @@ -335,7 +352,7 @@ void FieldManager::registration_ends () if (groups_to_bundle.size()>0) { using namespace ShortFieldTagsNames; - // A cluster is a pair + // A cluster is a list of names of groups in the cluster using cluster_type = std::list; // Determine if two lists have elements in common (does not compute the intersection) @@ -381,7 +398,7 @@ void FieldManager::registration_ends () } // Now we have clusters. For each cluster, build the list of lists, and call - // the contiguous_superset method. + // the contiguous_superset utility. for (auto& cluster : clusters) { using LOL_t = std::list>; @@ -502,13 +519,13 @@ void FieldManager::registration_ends () // Figure out the layout of the fields in this cluster, // and make sure they all have the same layout LayoutType lt = LayoutType::Invalid; - std::shared_ptr f_layout; + FieldLayout f_layout = FieldLayout::invalid(); for (const auto& fname : cluster_ordered_fields) { const auto& f = m_fields.at(fname); const auto& id = f->get_header().get_identifier(); if (lt==LayoutType::Invalid) { - f_layout = id.get_layout_ptr(); - lt = get_layout_type(f_layout->tags()); + f_layout = id.get_layout(); + lt = get_layout_type(f_layout.tags()); } else { EKAT_REQUIRE_MSG (lt==get_layout_type(id.get_layout().tags()), "Error! Found a group to bundle containing fields with different layouts.\n" @@ -525,7 +542,7 @@ void FieldManager::registration_ends () if (lt==LayoutType::Scalar2D) { c_layout = m_grid->get_2d_vector_layout(CMP,cluster_ordered_fields.size()); } else { - c_layout = m_grid->get_3d_vector_layout(f_layout->tags().back()==LEV,CMP,cluster_ordered_fields.size()); + c_layout = m_grid->get_3d_vector_layout(f_layout.tags().back()==LEV,CMP,cluster_ordered_fields.size()); } // The units for the bundled field are nondimensional, cause checking whether @@ -711,15 +728,16 @@ void FieldManager::clean_up() { void FieldManager::add_field (const Field& f) { // This method has a few restrictions on the input field. - EKAT_REQUIRE_MSG (m_repo_state==RepoState::Closed, + EKAT_REQUIRE_MSG (m_repo_state==RepoState::Closed or m_repo_state==RepoState::Clean, "Error! The method 'add_field' can only be called on a closed repo.\n"); EKAT_REQUIRE_MSG (f.is_allocated(), "Error! The method 'add_field' requires the input field to be already allocated.\n"); - EKAT_REQUIRE_MSG (f.get_header().get_identifier().get_grid_name()==m_grid->name(), - "Error! Input field to 'add_field' is defined on a grid different from the one stored.\n" + EKAT_REQUIRE_MSG (m_grid->is_valid_layout(f.get_header().get_identifier().get_layout()), + "Error! Input field to 'add_field' has a layout not compatible with the stored grid.\n" + " - input field name : " + f.name() + "\n" " - field manager grid: " + m_grid->name() + "\n" - " - input field grid: " + f.get_header().get_identifier().get_grid_name() + "\n"); - EKAT_REQUIRE_MSG (not has_field(f.get_header().get_identifier().name()), + " - input field layout: " + to_string(f.get_header().get_identifier().get_layout()) + "\n"); + EKAT_REQUIRE_MSG (not has_field(f.name()), "Error! The method 'add_field' requires the input field to not be already existing.\n" " - field name: " + f.get_header().get_identifier().name() + "\n"); EKAT_REQUIRE_MSG (f.get_header().get_tracking().get_groups_info().size()==0 || @@ -732,6 +750,8 @@ void FieldManager::add_field (const Field& f) { // All good, add the field to the repo m_fields[f.get_header().get_identifier().name()] = std::make_shared(f); + + m_repo_state = RepoState::Closed; } std::shared_ptr diff --git a/components/eamxx/src/share/field/field_manager.hpp b/components/eamxx/src/share/field/field_manager.hpp index e428f6273504..ad0c8bc47d8a 100644 --- a/components/eamxx/src/share/field/field_manager.hpp +++ b/components/eamxx/src/share/field/field_manager.hpp @@ -82,15 +82,13 @@ class FieldManager { // Query for a particular field or group of fields bool has_field (const std::string& name) const { return m_fields.find(name)!=m_fields.end(); } - bool has_field (const identifier_type& id) const; bool has_group (const std::string& name) const { return m_field_groups.find(name)!=m_field_groups.end(); } + const FieldIdentifier& get_field_id (const std::string& name) const; Field get_field (const std::string& name) const; - Field get_field (const identifier_type& id) const; - - // Unlike the previous two, these are allowed even if registration is ongoing - std::shared_ptr get_field_ptr(const std::string& name) const; - std::shared_ptr get_field_ptr(const identifier_type& id) const; + Field get_field (const identifier_type& id) const { return get_field(id.name()); } + Field& get_field (const std::string& name); + Field& get_field (const identifier_type& id) { return get_field(id.name()); } FieldGroup get_field_group (const std::string& name) const; @@ -105,6 +103,10 @@ class FieldManager { protected: + // These are allowed even if registration is ongoing + std::shared_ptr get_field_ptr(const std::string& name) const; + std::shared_ptr get_field_ptr(const identifier_type& id) const; + void pre_process_group_requests (); // The state of the repository @@ -130,6 +132,11 @@ class FieldManager { // The grid where the fields in this FM live std::shared_ptr m_grid; + + // If some fields are registered with incomplete FID (just name and grid), + // we 'skip' them, hoping that some other request will contain the right specs. + // If no complete request is given for that field, we need to error out + std::list> m_incomplete_requests; }; } // namespace scream diff --git a/components/eamxx/src/share/field/field_request.hpp b/components/eamxx/src/share/field/field_request.hpp index a772a7945248..bca9c46f4da6 100644 --- a/components/eamxx/src/share/field/field_request.hpp +++ b/components/eamxx/src/share/field/field_request.hpp @@ -251,12 +251,26 @@ struct FieldRequest { parent_name = parent.fid.name(); } + FieldRequest (const std::string& field_name, const std::string& grid_name, + const std::list& groups = {}, const int ps = 1) + : FieldRequest (incomplete_fid(field_name,grid_name),groups,ps) + { + incomplete = true; + } + + static FieldIdentifier + incomplete_fid (const std::string& field_name, const std::string& grid_name) + { + return FieldIdentifier(field_name,FieldLayout::invalid(),Units::invalid(),grid_name,DataType::Invalid); + } + // Data FieldIdentifier fid; int pack_size; std::list groups; SubviewInfo subview_info; std::string parent_name; + bool incomplete = false; }; // In order to use FieldRequest in std sorted containers (like std::set), diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index 6dd12360432e..a96f781a465c 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -38,7 +38,9 @@ enum class FieldTag { ShortWaveBand, ShortWaveGpoint, LongWaveBand, - LongWaveGpoint + LongWaveGpoint, + IsccpTau, + IsccpPrs }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -47,6 +49,7 @@ enum class FieldTag { // using enum FieldTag; namespace ShortFieldTagsNames { + constexpr auto INV = FieldTag::Invalid; constexpr auto EL = FieldTag::Element; constexpr auto COL = FieldTag::Column; constexpr auto GP = FieldTag::GaussPoint; @@ -60,6 +63,8 @@ namespace ShortFieldTagsNames { constexpr auto LWBND = FieldTag::LongWaveBand; constexpr auto SWGPT = FieldTag::ShortWaveGpoint; constexpr auto LWGPT = FieldTag::LongWaveGpoint; + constexpr auto ISCCPTAU = FieldTag::IsccpTau; + constexpr auto ISCCPPRS = FieldTag::IsccpPrs; } inline std::string e2str (const FieldTag ft) { @@ -106,6 +111,12 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::LongWaveGpoint: name = "lwgpt"; break; + case FieldTag::IsccpTau: + name = "ISCCPTAU"; + break; + case FieldTag::IsccpPrs: + name = "ISCCPPRS"; + break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/src/share/field/field_tracking.cpp b/components/eamxx/src/share/field/field_tracking.cpp index 7bf5f71d657e..a42650069fc6 100644 --- a/components/eamxx/src/share/field/field_tracking.cpp +++ b/components/eamxx/src/share/field/field_tracking.cpp @@ -31,6 +31,12 @@ void FieldTracking::update_time_stamp (const TimeStamp& ts) { } } +void FieldTracking::invalidate_time_stamp () +{ + // Reset the time stamp to an invalid time stamp + m_time_stamp = util::TimeStamp(); +} + void FieldTracking::set_accum_start_time (const TimeStamp& t_start) { EKAT_REQUIRE_MSG (not m_time_stamp.is_valid() || m_time_stamp<=t_start, "Error! Accumulation start time is older than current timestamp of the field.\n"); diff --git a/components/eamxx/src/share/field/field_tracking.hpp b/components/eamxx/src/share/field/field_tracking.hpp index 1dc287ee26f9..6a8f52a7f0da 100644 --- a/components/eamxx/src/share/field/field_tracking.hpp +++ b/components/eamxx/src/share/field/field_tracking.hpp @@ -60,6 +60,7 @@ class FieldTracking : public FamilyTracking { // NOTE: if the field has 'children' (see FamilyTracking), their ts will be updated too. // However, if the field has a 'parent' (see FamilyTracking), the parent's ts will not be updated. void update_time_stamp (const TimeStamp& ts); + void invalidate_time_stamp (); // Set/get accumulation interval start void set_accum_start_time (const TimeStamp& ts); @@ -73,7 +74,7 @@ class FieldTracking : public FamilyTracking { // Tracking the updates of the field TimeStamp m_time_stamp; - // For accummulated vars, the time where the accummulation started + // For accumulated vars, the time where the accumulation started TimeStamp m_accum_start; ci_string m_accum_type; diff --git a/components/eamxx/src/share/field/field_utils.hpp b/components/eamxx/src/share/field/field_utils.hpp index e0a9f83e4205..b58b60f40bfb 100644 --- a/components/eamxx/src/share/field/field_utils.hpp +++ b/components/eamxx/src/share/field/field_utils.hpp @@ -43,6 +43,74 @@ void randomize (const Field& f, Engine& engine, PDF&& pdf) impl::randomize(f,engine,pdf); } +// Compute a random perturbation of a field for all view entries whose +// level index satisfies the mask. +// Input: +// - f: Field to perturbed. Required to have level midpoint +// tag as last dimension. +// - engine: Random number engine. +// - pdf: Random number distribution where a random value (say, +// pertval) is taken s.t. +// field_view(i0,...,iN) *= (1 + pertval) +// - base_seed: Seed used for creating the engine input. +// - level_mask: Mask (size of the level dimension of f) where f(i0,...,k) is +// perturbed if level_mask(k)=true +// - dof_gids: Field containing global DoF IDs for columns of f (if applicable) +template +void perturb (const Field& f, + Engine& engine, + PDF&& pdf, + const int base_seed, + const MaskType& level_mask, + const Field& dof_gids = Field()) +{ + EKAT_REQUIRE_MSG(f.is_allocated(), + "Error! Cannot perturb the values of a field not yet allocated.\n"); + + // Deduce scalar type from pdf + using ST = decltype(pdf(engine)); + + // Check compatibility between PDF and field data type + const auto data_type = f.data_type(); + EKAT_REQUIRE_MSG((std::is_same_v && data_type==DataType::IntType) or + (std::is_same_v && data_type==DataType::FloatType) or + (std::is_same_v && data_type==DataType::DoubleType), + "Error! Field data type incompatible with input PDF.\n"); + + using namespace ShortFieldTagsNames; + const auto& fl = f.get_header().get_identifier().get_layout(); + + // Field we are perturbing should have a level dimension, + // and it is required to be the last dimension + EKAT_REQUIRE_MSG(fl.rank()>0 && + (fl.tags().back() == LEV || fl.tags().back() == ILEV), + "Error! Trying to perturb field \""+f.name()+"\", but field " + "does not have LEV or ILEV as last dimension.\n" + " - field name: " + f.name() + "\n" + " - field layout: " + to_string(fl) + "\n"); + + if (fl.has_tag(COL)) { + // If field has a column dimension, it should be the first dimension + EKAT_REQUIRE_MSG(fl.tag(0) == COL, + "Error! Trying to perturb field \""+f.name()+"\", but field " + "does not have COL as first dimension.\n" + " - field name: " + f.name() + "\n" + " - field layout: " + to_string(fl) + "\n"); + + const auto& dof_gids_fl = dof_gids.get_header().get_identifier().get_layout(); + EKAT_REQUIRE_MSG(dof_gids_fl.dim(0) == fl.dim(COL), + "Error! Field of DoF GIDs should have the same size as " + "perturbed field's column dimension.\n" + " - dof_gids dim: " + std::to_string(dof_gids_fl.dim(0)) + "\n" + " - field name: " + f.name() + "\n" + " - field layout: " + to_string(fl) + "\n"); + EKAT_REQUIRE_MSG(dof_gids.data_type() == DataType::IntType, + "Error! DoF GIDs field must have \"int\" as data type.\n"); + } + + impl::perturb(f, engine, pdf, base_seed, level_mask, dof_gids); +} + template ST frobenius_norm(const Field& f, const ekat::Comm* comm = nullptr) { diff --git a/components/eamxx/src/share/field/field_utils_impl.hpp b/components/eamxx/src/share/field/field_utils_impl.hpp index 877f24d185fa..456dab64cc23 100644 --- a/components/eamxx/src/share/field/field_utils_impl.hpp +++ b/components/eamxx/src/share/field/field_utils_impl.hpp @@ -34,8 +34,8 @@ bool views_are_equal(const Field& f1, const Field& f2, const ekat::Comm* comm) switch (l1.rank()) { case 1: { - auto v1 = f1.template get_view(); - auto v2 = f2.template get_view(); + auto v1 = f1.template get_strided_view(); + auto v2 = f2.template get_strided_view(); for (int i=0; i(); + v() = pdf(engine); + } + break; case 1: { auto v = f.template get_view(); @@ -208,6 +214,71 @@ void randomize (const Field& f, Engine& engine, PDF&& pdf) f.sync_to_dev(); } +template +void perturb (const Field& f, + Engine& engine, + PDF&& pdf, + const unsigned int base_seed, + const MaskType& level_mask, + const Field& dof_gids) +{ + const auto& fl = f.get_header().get_identifier().get_layout(); + + // Check to see if field has a column dimension + using namespace ShortFieldTagsNames; + const bool has_column_dim = fl.has_tag(COL); + + if (has_column_dim) { + // Because Column is the partitioned dimension, we must reset the + // RNG seed to be the same on every column so that a column will + // have the same value no matter where it exists in an MPI rank's + // set of local columns. + const auto gids = dof_gids.get_view(); + + // Create a field to store perturbation values with layout + // the same as f, but stripped of column and level dimension. + auto perturb_fl = fl.strip_dim(COL).strip_dim(LEV); + FieldIdentifier perturb_fid("perturb_field", perturb_fl, ekat::units::Units::nondimensional(), ""); + Field perturb_f(perturb_fid); + perturb_f.allocate_view(); + + // Loop through columns as reset RNG seed based on GID of column + for (auto icol=0; icol ST frobenius_norm(const Field& f, const ekat::Comm* comm) { @@ -223,7 +294,7 @@ ST frobenius_norm(const Field& f, const ekat::Comm* comm) switch (fl.rank()) { case 1: { - auto v = f.template get_view(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i(); + auto v = f.template get_view(); for (int i=0; i2 - // So if the input field is 2d and the index to subview at is along - // the 2nd dim, we must stop recursion, and print the field manually - // extracting the last slice. // // We keep all the tags/indices we slice away, since we need them at // the end of recursion when we print the info of the field location. @@ -691,7 +756,7 @@ void print_field_hyperslab (const Field& f, { dims_str[dims_left[0]] = ":"; out << " " << f.name() << "(" << ekat::join(dims_str,",") << ")"; - auto v = f.get_view(); + auto v = f.get_strided_view(); for (int i=0; i\n" " - loc indices : (" + ekat::join(indices,",") + ")\n"); - // It's always safe to take subview along 1st dim. For 2nd dim, we can - // only do it if the rank is 3+ - if (idim==0 || layout.rank()>2) { - // Slice field, and recurse. - auto sub_f = f.subfield(idim,idx); - return print_field_hyperslab(sub_f,tags,indices,out,orig_rank,curr_idx+1); - } else { - // It must be that we have one last slicing to do, - // along the 2nd dimension of a rank2 field. - EKAT_REQUIRE_MSG (curr_idx==(tags.size()-1) && layout.rank()==2, - "Error! Unexpected inputs in print_field_hyperslab.\n" - " - field name : " + f.name() + "\n" - " - field layout: " + to_string(layout) + "\n" - " - curr tag : " + e2str(tag) + "\n" - " - loc tags : <" + ekat::join(tags,",") + ">\n" - " - loc indices: (" + ekat::join(indices,",") +")\n"); - - const auto& orig_layout = get_orig_header()->get_identifier().get_layout(); - const auto dims_left = get_dims_left(orig_layout); - auto dims_str = get_dims_str(orig_layout); - - // Although the view we extract still has rank2, get_dims_[str|left] only - // use the original layout and the loc tags/indices, so it already filled - // all the dims string corresponding to the input loc tags/indices. - // The only dim left is the current 1st dim of the field. - // In other words, even though we have a rank2 field, we should have ended - // with a rank 1 field (if Kokkos had allowed us to take the subview) - dims_str[dims_left[0]] = ":"; - - out << " " << f.name() << to_string(orig_layout) << "\n\n"; - f.sync_to_host(); - auto v = f.get_view(); - out << " " << f.name() << "(" << ekat::join(dims_str,",") << ")"; - for (int i=0; i(sub_f,tags,indices,out,orig_rank,curr_idx+1); } } diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 082b9c1e4043..a345df2f7b19 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -2,6 +2,8 @@ #include "share/field/field_utils.hpp" +#include + #include #include #include @@ -28,6 +30,32 @@ AbstractGrid (const std::string& name, // This grid name is also an alias m_aliases.push_back(m_name); + + // Ensure each grid object gets a different id + static int counter = 0; + m_unique_grid_id = counter; + ++counter; +} + +AbstractGrid:: +AbstractGrid (const std::string& name, + const GridType type, + const int num_local_dofs, + const int num_global_dofs, + const int num_vertical_lev, + const ekat::Comm& comm) + : AbstractGrid(name,type,num_local_dofs,num_vertical_lev,comm) +{ + m_num_global_dofs = num_global_dofs; +#ifndef NDEBUG + int max_nldofs = m_num_local_dofs; + m_comm.all_reduce(&max_nldofs,1,MPI_MAX); + EKAT_REQUIRE_MSG (max_nldofs<=m_num_global_dofs, + "Error! The number of global dof is smaller than the local number of dofs on some ranks.\n" + " - grid name: " + name + "\n" + " - num global dofs: " + std::to_string(m_num_global_dofs) + "\n" + " - max num local dofs: " + std::to_string(max_nldofs) + "\n"); +#endif } void AbstractGrid::add_alias (const std::string& alias) @@ -47,63 +75,106 @@ get_vertical_layout (const bool midpoints) const } bool AbstractGrid::is_unique () const { - // Get a copy of gids on host. CAREFUL: do not use the stored dofs, - // since we need to sort dofs in order to call unique, and we don't - // want to alter the order of gids in this grid. - auto dofs = m_dofs_gids.clone(); - auto dofs_h = dofs.get_view(); - - std::sort(dofs_h.data(),dofs_h.data()+m_num_local_dofs); - auto unique_end = std::unique(dofs_h.data(),dofs_h.data()+m_num_local_dofs); - - int locally_unique = unique_end==(dofs_h.data()+m_num_local_dofs); - int unique; - m_comm.all_reduce(&locally_unique,&unique,1,MPI_PROD); - if (unique==0) { - return false; - } - - // Each rank has unique gids locally. Now it's time to verify if they are also globally unique. - int max_dofs; - m_comm.all_reduce(&m_num_local_dofs,&max_dofs,1,MPI_MAX); - std::vector gids(max_dofs,-1); - int unique_gids = 1; - - for (int pid=0; pid(); + + std::sort(dofs_h.data(),dofs_h.data()+m_num_local_dofs); + auto unique_end = std::unique(dofs_h.data(),dofs_h.data()+m_num_local_dofs); + + int locally_unique = unique_end==(dofs_h.data()+m_num_local_dofs); + int unique; + m_comm.all_reduce(&locally_unique,&unique,1,MPI_PROD); + if (unique==0) { + return false; } - int ndofs = m_num_local_dofs; - m_comm.broadcast(&ndofs,1,pid); - m_comm.broadcast(gids.data(),ndofs,pid); - - int my_unique_gids = 1; - if (pid!=m_comm.rank()) { - // Checking two sorted arrays of length m and n for elements in common is O(m+n) ops. - int i=0, j=0; - while (igids[j]) { - ++j; - } else { - // Found a match. We can stop here - my_unique_gids = 0; - break; + // Each rank has unique gids locally. Now it's time to verify if they are also globally unique. + int max_dofs; + m_comm.all_reduce(&m_num_local_dofs,&max_dofs,1,MPI_MAX); + std::vector gids(max_dofs,-1); + int unique_gids = 1; + + for (int pid=0; pidgids[j]) { + ++j; + } else { + // Found a match. We can stop here + my_unique_gids = 0; + break; + } } } + m_comm.all_reduce(&my_unique_gids,&unique_gids,1,MPI_PROD); + if (unique_gids==0) { + break; + } } - m_comm.all_reduce(&my_unique_gids,&unique_gids,1,MPI_PROD); - if (unique_gids==0) { - break; - } + return unique_gids==1; + }; + + if (not m_is_unique_computed) { + m_is_unique = compute_is_unique(); + m_is_unique_computed = true; } + return m_is_unique; +} - return unique_gids; +bool AbstractGrid:: +is_valid_layout (const FieldLayout& layout) const +{ + using namespace ShortFieldTagsNames; + + const auto lt = get_layout_type(layout.tags()); + if (lt==LayoutType::Scalar0D or lt==LayoutType::Vector0D) { + // 0d layouts are compatible with any grid + // Let's return true early to avoid segfautls below + return true; + } + const bool midpoints = layout.tags().back()==LEV; + const bool is_vec = layout.is_vector_layout(); + const int vec_dim = is_vec ? layout.dims()[layout.get_vector_dim()] : 0; + const auto vec_tag = is_vec ? layout.get_vector_tag() : INV; + + switch (lt) { + case LayoutType::Scalar1D: [[fallthrough]]; + case LayoutType::Vector1D: + // 1d layouts need the right number of levels + return layout.dims().back() == m_num_vert_levs or + layout.dims().back() == (m_num_vert_levs+1); + case LayoutType::Scalar2D: + return layout==get_2d_scalar_layout(); + case LayoutType::Vector2D: + return layout==get_2d_vector_layout(vec_tag,vec_dim); + case LayoutType::Scalar3D: + return layout==get_3d_scalar_layout(midpoints); + case LayoutType::Vector3D: + return layout==get_3d_vector_layout(midpoints,vec_tag,vec_dim); + default: + // Anything else is probably no + return false; + } } auto AbstractGrid:: @@ -196,8 +267,19 @@ AbstractGrid::get_geometry_data_names () const void AbstractGrid::reset_num_vertical_lev (const int num_vertical_lev) { m_num_vert_levs = num_vertical_lev; - // TODO: when the PR storing geo data as Field goes in, you should - // invalidate all geo data whose FieldLayout contains LEV/ILEV + using namespace ShortFieldTagsNames; + + // Loop over geo data. If they have the LEV or ILEV tag, they are + // no longer valid, so we must erase them. + for (auto it=m_geo_fields.cbegin(); it!=m_geo_fields.cend(); ) { + const auto& fl = it->second.get_header().get_identifier().get_layout(); + const auto has_lev = fl.has_tag(LEV) or fl.has_tag(ILEV); + if (has_lev) { + it = m_geo_fields.erase(it); + } else { + ++it; + } + } } std::vector @@ -303,6 +385,73 @@ get_owners (const gid_view_h& gids) const return result; } +void AbstractGrid:: +get_remote_pids_and_lids (const gid_view_h& gids, + std::vector& pids, + std::vector& lids) const +{ + const auto& comm = get_comm(); + int num_gids_in = gids.size(); + + pids.resize(num_gids_in,-1); + lids.resize(num_gids_in,-1); + + int num_found = 0; + + // We may have repeated gids. In that case, we want to update + // the pids/lids arrays at all indices corresponding to the same gid + std::map> gid2idx; + for (int i=0; i(); + gid_type* data; + std::vector pid_gids; + for (int pid=0; pidsecond) { + EKAT_REQUIRE_MSG (pids[idx]==-1, + "Error! Found a GID with multiple owners.\n" + " - owner 1: " + std::to_string(pids[idx]) + "\n" + " - owner 2: " + std::to_string(pid) + "\n"); + pids[idx] = pid; + lids[idx] = i; + } + ++num_found; + } + } + } + EKAT_REQUIRE_MSG (num_found==num_unique_gids, + "Error! Could not locate the owner of one of the input GIDs.\n" + " - rank: " + std::to_string(comm.rank()) + "\n" + " - num found: " + std::to_string(num_found) + "\n" + " - num unique gids in: " + std::to_string(num_unique_gids) + "\n"); +} + void AbstractGrid::create_dof_fields (const int scalar2d_layout_rank) { using namespace ShortFieldTagsNames; @@ -320,6 +469,17 @@ void AbstractGrid::create_dof_fields (const int scalar2d_layout_rank) m_lid_to_idx.allocate_view(); } +auto AbstractGrid::get_gid2lid_map () const + -> std::map +{ + std::map m; + auto gids_h = get_dofs_gids().get_view(); + for (int i=0; i const int num_vertical_lev, const ekat::Comm& comm); + AbstractGrid (const std::string& name, + const GridType type, + const int num_local_dofs, + const int num_global_dofs, + const int num_vertical_lev, + const ekat::Comm& comm); + virtual ~AbstractGrid () = default; // Grid description utilities @@ -75,6 +82,9 @@ class AbstractGrid : public ekat::enable_shared_from_this // Whether this grid contains unique dof GIDs bool is_unique () const; + // Check if the input layout is compatible with this grid + bool is_valid_layout (const FieldLayout& layout) const; + // When running with multiple ranks, fields are partitioned across ranks along this FieldTag virtual FieldTag get_partitioned_dim_tag () const = 0; @@ -138,6 +148,16 @@ class AbstractGrid : public ekat::enable_shared_from_this return get_owners(gids_v); } + void get_remote_pids_and_lids (const gid_view_h& gids, + std::vector& pids, + std::vector& lids) const; + void get_remote_pids_and_lids (const std::vector& gids, + std::vector& pids, + std::vector& lids) const { + gid_view_h gids_v(gids.data(),gids.size()); + get_remote_pids_and_lids(gids_v,pids,lids); + } + // Derived classes can override these methods to verify that the // dofs have been set to something that satisfies any requirement of the grid type. virtual bool check_valid_dofs() const { return true; } @@ -154,6 +174,10 @@ class AbstractGrid : public ekat::enable_shared_from_this // NOTE: we'd need setter/getter for this, so we might as well make it public std::string m_short_name = ""; + int get_unique_grid_id () const { return m_unique_grid_id; } + + std::map get_gid2lid_map () const; + protected: void copy_data (const AbstractGrid& src, const bool shallow = true); @@ -168,6 +192,8 @@ class AbstractGrid : public ekat::enable_shared_from_this GridType m_type; std::string m_name; + int m_unique_grid_id; + std::vector m_aliases; std::map m_special_tag_names; @@ -184,6 +210,10 @@ class AbstractGrid : public ekat::enable_shared_from_this mutable gid_type m_global_min_dof_gid = std::numeric_limits::max(); mutable gid_type m_global_max_dof_gid = -std::numeric_limits::max(); + // The fcn is_unique is expensive, so we lazy init this at the first call. + mutable bool m_is_unique; + mutable bool m_is_unique_computed = false; + // The map lid->idx Field m_lid_to_idx; diff --git a/components/eamxx/src/share/grid/grid_import_export.cpp b/components/eamxx/src/share/grid/grid_import_export.cpp new file mode 100644 index 000000000000..5f87e40598d6 --- /dev/null +++ b/components/eamxx/src/share/grid/grid_import_export.cpp @@ -0,0 +1,174 @@ +#include "grid_import_export.hpp" + +#include "share/field/field_utils.hpp" + +namespace scream +{ + +GridImportExport:: +GridImportExport (const std::shared_ptr& unique, + const std::shared_ptr& overlapped) +{ + EKAT_REQUIRE_MSG (unique!=nullptr, "Error! Input unique grid pointer is null.\n"); + EKAT_REQUIRE_MSG (overlapped!=nullptr, "Error! Input overlapped grid pointer is null.\n"); + + EKAT_REQUIRE_MSG (unique->is_unique(), + "Error! GridImportExport unique grid is not unique.\n"); + + using gid_type = AbstractGrid::gid_type; + + m_unique = unique; + m_overlapped = overlapped; + m_comm = unique->get_comm(); + + // Note: we can't use the gids views from the grids, since we need to pass pointers + // to MPI bcast routines, which require pointers to nonconst data. + // Hence, create a clone, and grab a non-const pointer from it. + const auto unique_gids_f = unique->get_dofs_gids().clone(); + const auto overlap_gids_f = overlapped->get_dofs_gids().clone(); + const auto gids = unique_gids_f.get_view(); + const auto ov_gids = overlap_gids_f.get_view(); + + int num_ov_gids = ov_gids.size(); + + gid_type* data; + std::vector pid_gids; + std::map> pid2lids; + + // ------------------ Create import structures ----------------------- // + + // Resize output + m_import_lids = decltype(m_import_lids)("",num_ov_gids); + m_import_pids = decltype(m_import_pids)("",num_ov_gids); + + m_import_lids_h = Kokkos::create_mirror_view(m_import_lids); + m_import_pids_h = Kokkos::create_mirror_view(m_import_pids); + Kokkos::deep_copy(m_import_pids_h,-1); + + // We may have repeated gids. In that case, we want to update + // the pids/lids arrays at all indices corresponding to the same gid + // std::map> gid2idx; + // for (int i=0; iget_gid2lid_map(); + + // Let each rank bcast its src gids, so that other procs can + // check against their dst grid + int num_imports = 0; + for (int pid=0; pidsecond); + ++num_imports; + } + } + } + EKAT_REQUIRE_MSG (num_ov_gids==num_imports, + "Error! Could not locate the owner of one of the dst grid GIDs.\n" + " - rank: " + std::to_string(m_comm.rank()) + "\n" + " - num found: " + std::to_string(num_imports) + "\n" + " - num dst gids: " + std::to_string(num_ov_gids) + "\n"); + for (int pid=0,pos=0; pidget_gid2lid_map(); + + // Let each rank bcast its src gids, so that other procs can + // check against their dst grid + // Note: we don't know a priori how many PIDs will need each + // of our dofs, so we cannot insert in the export pids/lids views yet, + // and must use a temporary map to store results + int num_exports = 0; + pid2lids.clear(); + for (int pid=0; pidsecond); + ++num_exports; + } + } + + // IMPORTANT! When building the import data, within each PID, we order + // the list of imports according to the *remote* ordering. In order for + // p2p messages to be consistent, the export data must order the + // list of exports according to the *local* ordring. + std::sort(pid2lids[pid].begin(),pid2lids[pid].end()); + } + + m_export_pids = view_1d("",num_exports); + m_export_lids = view_1d("",num_exports); + m_export_lids_h = Kokkos::create_mirror_view(m_export_lids); + m_export_pids_h = Kokkos::create_mirror_view(m_export_pids); + for (int pid=0,pos=0; pid("",m_comm.size()); + m_num_exports_per_pid_h = Kokkos::create_mirror_view(m_num_exports_per_pid); + for (size_t i=0; i("",m_comm.size()); + m_num_imports_per_pid_h = Kokkos::create_mirror_view(m_num_imports_per_pid); + for (size_t i=0; i +#include // We do some direct MPI calls +#include +#include +#include + +namespace scream +{ + +/* + * Import/Export data is used to figure out where data + * can be retrieved or sent (both in terms of remote + * rank and remote local id) when transferring between + * two grids. The terms import/export do not necessarily + * imply that we are getting/sending data. Rather, they + * have to do with which grid we're initiating the + * transfer from. Namely, + * - import: the grid we have data on is potentially + * non unique, and the target grid is unique + * - export: the grid we have data on is unique, and + * the target grid is potentially non unique + * Import/Export data plans can be used both for scattering + * and gathering data. The user can use the gather/scatter + * methods for this, but pay attention to their limitations: + * - they create send/recv requests at every call (no persistent requests) + * - they assume same data type on origin/target ranks + * - they operate on a particular input/output data ortanization, + * namely, data is organized as a map lid->vector + * - the above point implies data must be on Host + * These limitations imply that gather/scatter methods are only + * for ease of use in non-performance critical code. + * On the other hand, the import/export data (pids/lids) can + * be used both on host and device, for more efficient pack/unpack methods. + */ + +class GridImportExport { +public: + using KT = KokkosTypes; + template + using view_1d = typename KT::view_1d; + + GridImportExport (const std::shared_ptr& unique, + const std::shared_ptr& overlapped); + ~GridImportExport () = default; + + template + void scatter (const MPI_Datatype mpi_data_t, + const std::map>& src, + std::map>& dst) const; + + template + void gather (const MPI_Datatype mpi_data_t, + const std::map>& src, + std::map>& dst) const; + + view_1d num_exports_per_pid () const { return m_num_exports_per_pid; } + view_1d num_imports_per_pid () const { return m_num_imports_per_pid; } + + view_1d::HostMirror num_exports_per_pid_h () const { return m_num_exports_per_pid_h; } + view_1d::HostMirror num_imports_per_pid_h () const { return m_num_imports_per_pid_h; } + + view_1d import_pids () const { return m_import_pids; } + view_1d import_lids () const { return m_import_lids; } + view_1d export_pids () const { return m_export_pids; } + view_1d export_lids () const { return m_export_lids; } + + view_1d::HostMirror import_pids_h () const { return m_import_pids_h; } + view_1d::HostMirror import_lids_h () const { return m_import_lids_h; } + view_1d::HostMirror export_pids_h () const { return m_export_pids_h; } + view_1d::HostMirror export_lids_h () const { return m_export_lids_h; } + +protected: + + std::shared_ptr m_unique; + std::shared_ptr m_overlapped; + + // All these arrays are sorted by pid. That is, all imports + // for pid 1 come before imports for pid 2, and same for exports. + view_1d m_import_pids; + view_1d m_import_lids; + view_1d m_export_pids; + view_1d m_export_lids; + + view_1d::HostMirror m_import_pids_h; + view_1d::HostMirror m_import_lids_h; + view_1d::HostMirror m_export_pids_h; + view_1d::HostMirror m_export_lids_h; + + view_1d m_num_imports_per_pid; + view_1d m_num_exports_per_pid; + + view_1d::HostMirror m_num_imports_per_pid_h; + view_1d::HostMirror m_num_exports_per_pid_h; + + ekat::Comm m_comm; +}; + +// --------------------- IMPLEMENTATION ------------------------ // + +template +void GridImportExport:: +scatter (const MPI_Datatype mpi_data_t, + const std::map>& src, + std::map>& dst) const +{ + using gid_type = AbstractGrid::gid_type; + + std::vector send_req, recv_req; + + const int nexp = m_export_lids.size(); + const int nimp = m_import_lids.size(); + auto mpi_comm = m_comm.mpi_comm(); + + auto unique_gids_h = m_unique->get_dofs_gids().get_view(); + auto overlap_gids_h = m_overlapped->get_dofs_gids().get_view(); + + // 1. Communicate to the recv pids how many items per lid + // we need to send + std::vector send_count(m_export_lids_h.size(),0); + for (int i=0; i recv_count(m_import_lids_h.size(),0); + for (int i=0; i +void GridImportExport:: +gather (const MPI_Datatype mpi_data_t, + const std::map>& src, + std::map>& dst) const +{ + using gid_type = AbstractGrid::gid_type; + + std::vector send_req, recv_req; + + const int nexp = m_export_lids.size(); + const int nimp = m_import_lids.size(); + auto mpi_comm = m_comm.mpi_comm(); + + auto unique_gids_h = m_unique->get_dofs_gids().get_view(); + auto overlap_gids_h = m_overlapped->get_dofs_gids().get_view(); + + const int num_ov_gids = overlap_gids_h.size(); + + // 1. Communicate to the recv pids how many items per lid + // we need to send + std::vector send_count(num_ov_gids,0); + for (int i=0; i recv_count(m_export_lids_h.size(),0); + for (int i=0; i> recv_buf; + for (int i=0; i; - using grid_repo_type = std::map; - using remapper_type = AbstractRemapper; - using remapper_ptr_type = std::shared_ptr; + using grid_type = AbstractGrid; + using grid_ptr_type = std::shared_ptr; + using grid_repo_type = std::map; + using nonconstgrid_ptr_type = std::shared_ptr; + using nonconstgrid_repo_type = std::map; + using remapper_type = AbstractRemapper; + using remapper_ptr_type = std::shared_ptr; GridsManager () = default; virtual ~GridsManager () = default; @@ -33,6 +35,7 @@ class GridsManager virtual std::string name () const = 0; grid_ptr_type get_grid (const std::string& name) const; + nonconstgrid_ptr_type get_grid_nonconst (const std::string& name) const; // Check if the given grid has been built bool has_grid (const std::string& grid_name) const; @@ -52,12 +55,8 @@ class GridsManager const grid_repo_type& get_repo () const { return m_grids; } protected: - using nonconstgrid_ptr_type = std::shared_ptr; - using nonconstgrid_repo_type = std::map; void add_grid (nonconstgrid_ptr_type grid); - nonconstgrid_ptr_type get_grid_nonconst (const std::string& name) const; - void alias_grid (const std::string& grid_name, const std::string& grid_alias); virtual remapper_ptr_type diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index 965027008fd3..ed8e5bb2c254 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -20,6 +20,7 @@ MeshFreeGridsManager (const ekat::Comm& comm, const ekat::ParameterList& p) : m_params (p) , m_comm (comm) { + // Nothing else to do here } MeshFreeGridsManager::remapper_ptr_type @@ -33,82 +34,100 @@ do_create_remapper (const grid_ptr_type from_grid, void MeshFreeGridsManager:: build_grids () { - auto has_positive_int = [&](const std::string& n) -> bool { - return m_params.isParameter(n) && (m_params.get(n)>0); - }; - const bool build_pt = has_positive_int("number_of_global_columns"); - const bool build_se = has_positive_int("number_of_local_elements") && - has_positive_int("number_of_gauss_points"); - - const int num_vertical_levels = m_params.get("number_of_vertical_levels"); - - if (build_se) { - // Build a set of completely disconnected spectral elements. - const int num_local_elems = m_params.get("number_of_local_elements"); - const int num_gp = m_params.get("number_of_gauss_points"); - - // Create the grid - std::shared_ptr se_grid; - se_grid = std::make_shared("SE Grid",num_local_elems,num_gp,num_vertical_levels,m_comm); - se_grid->setSelfPointer(se_grid); - - // Set up the degrees of freedom. - auto dof_gids = se_grid->get_dofs_gids(); - auto lid2idx = se_grid->get_lid_to_idx_map(); - - auto host_dofs = dof_gids.template get_view(); - auto host_lid2idx = lid2idx.template get_view(); - - // Count unique local dofs. On all elems except the very last one (on rank N), - // we have num_gp*(num_gp-1) unique dofs; - int num_local_dofs = num_local_elems*num_gp*num_gp; - int offset = num_local_dofs*m_comm.rank(); - - for (int ie = 0; ie < num_local_elems; ++ie) { - for (int igp = 0; igp < num_gp; ++igp) { - for (int jgp = 0; jgp < num_gp; ++jgp) { - int idof = ie*num_gp*num_gp + igp*num_gp + jgp; - int gid = offset + idof; - host_dofs(idof) = gid; - host_lid2idx(idof, 0) = ie; - host_lid2idx(idof, 1) = igp; - host_lid2idx(idof, 2) = jgp; - } + const auto& names = m_params.get>("grids_names"); + for (const auto& gname : names) { + auto& params = m_params.sublist(gname); + const auto& type = params.get("type"); + if (type=="se_grid") { + build_se_grid (gname,params); + } else if (type=="point_grid") { + build_point_grid (gname,params); + } else { + EKAT_ERROR_MSG ( + "[MeshFreeGridsManager::build_grids] Unrecognized grid type.\n" + " - grid name: " + gname + "\n" + " - grid type: " + type + "\n" + " - valid types: se_grid, point_grid\n"); + } + const auto& aliases = params.get>("aliases",{}); + for (const auto& n : aliases) { + this->alias_grid(gname, n); + } + } +} + +void MeshFreeGridsManager:: +build_se_grid (const std::string& name, ekat::ParameterList& params) +{ + // Build a set of completely disconnected spectral elements. + const int num_local_elems = params.get("number_of_local_elements"); + const int num_gp = params.get("number_of_gauss_points"); + const int num_vertical_levels = params.get("number_of_vertical_levels"); + + // Create the grid + std::shared_ptr se_grid; + se_grid = std::make_shared(name,num_local_elems,num_gp,num_vertical_levels,m_comm); + se_grid->setSelfPointer(se_grid); + + // Set up the degrees of freedom. + auto dof_gids = se_grid->get_dofs_gids(); + auto lid2idx = se_grid->get_lid_to_idx_map(); + + auto host_dofs = dof_gids.template get_view(); + auto host_lid2idx = lid2idx.template get_view(); + + // Count unique local dofs. On all elems except the very last one (on rank N), + // we have num_gp*(num_gp-1) unique dofs; + int num_local_dofs = num_local_elems*num_gp*num_gp; + int offset = num_local_dofs*m_comm.rank(); + + for (int ie = 0; ie < num_local_elems; ++ie) { + for (int igp = 0; igp < num_gp; ++igp) { + for (int jgp = 0; jgp < num_gp; ++jgp) { + int idof = ie*num_gp*num_gp + igp*num_gp + jgp; + int gid = offset + idof; + host_dofs(idof) = gid; + host_lid2idx(idof, 0) = ie; + host_lid2idx(idof, 1) = igp; + host_lid2idx(idof, 2) = jgp; } } + } - // Sync to device - dof_gids.sync_to_dev(); - lid2idx.sync_to_dev(); + // Sync to device + dof_gids.sync_to_dev(); + lid2idx.sync_to_dev(); - se_grid->m_short_name = "se"; - add_geo_data(se_grid); + se_grid->m_short_name = "se"; + add_geo_data(se_grid); - add_grid(se_grid); - } - if (build_pt) { - const int num_global_cols = m_params.get("number_of_global_columns"); - auto pt_grid = create_point_grid("Point Grid",num_global_cols,num_vertical_levels,m_comm); + add_grid(se_grid); +} - const auto units = ekat::units::Units::nondimensional(); +void MeshFreeGridsManager:: +build_point_grid (const std::string& name, ekat::ParameterList& params) +{ + const int num_global_cols = params.get("number_of_global_columns"); + const int num_vertical_levels = params.get("number_of_vertical_levels"); + auto pt_grid = create_point_grid(name,num_global_cols,num_vertical_levels,m_comm); - auto area = pt_grid->create_geometry_data("area", pt_grid->get_2d_scalar_layout(), units); + const auto units = ekat::units::Units::nondimensional(); - // Estimate cell area for a uniform grid by taking the surface area - // of the earth divided by the number of columns. Note we do this in - // units of radians-squared. - using PC = scream::physics::Constants; - const Real pi = PC::Pi; - const Real cell_area = 4.0*pi/num_global_cols; - area.deep_copy(cell_area); - area.sync_to_host(); + auto area = pt_grid->create_geometry_data("area", pt_grid->get_2d_scalar_layout(), units); - add_geo_data(pt_grid); - pt_grid->m_short_name = "pt"; + // Estimate cell area for a uniform grid by taking the surface area + // of the earth divided by the number of columns. Note we do this in + // units of radians-squared. + using PC = scream::physics::Constants; + const Real pi = PC::Pi; + const Real cell_area = 4.0*pi/num_global_cols; + area.deep_copy(cell_area); + area.sync_to_host(); - add_grid(pt_grid); - this->alias_grid("Point Grid", "Physics"); - } + add_geo_data(pt_grid); + pt_grid->m_short_name = "pt"; + + add_grid(pt_grid); } void MeshFreeGridsManager:: @@ -254,10 +273,23 @@ create_mesh_free_grids_manager (const ekat::Comm& comm, const int num_local_elem const int num_global_cols) { ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns",num_global_cols); - gm_params.set("number_of_local_elements",num_local_elems); - gm_params.set("number_of_gauss_points",num_gp); - gm_params.set("number_of_vertical_levels",num_vertical_levels); + std::vector grids_names; + if (num_local_elems>=2) { + grids_names.push_back("SE Grid"); + auto& pl = gm_params.sublist("SE Grid"); + pl.set("type",std::string("se_grid")); + pl.set("number_of_local_elements",num_local_elems); + pl.set("number_of_gauss_points",num_gp); + pl.set("number_of_vertical_levels",num_vertical_levels); + } + if (num_global_cols>0) { + grids_names.push_back("Point Grid"); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type",std::string("point_grid")); + pl.set("number_of_global_columns",num_global_cols); + pl.set("number_of_vertical_levels",num_vertical_levels); + } + gm_params.set("grids_names",grids_names); auto gm = create_mesh_free_grids_manager(comm,gm_params); return gm; } diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.hpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.hpp index 0d3173b17f50..078b5fa2d4a5 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.hpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.hpp @@ -31,6 +31,11 @@ class MeshFreeGridsManager : public GridsManager protected: + void build_se_grid (const std::string& name, + ekat::ParameterList& params); + void build_point_grid (const std::string& name, + ekat::ParameterList& params); + void add_geo_data (const nonconstgrid_ptr_type& grid) const; std::string get_reference_grid_name () const { @@ -55,6 +60,7 @@ create_mesh_free_grids_manager (const ekat::Comm& comm, const ekat::ParameterLis return std::make_shared(comm,p); } +// Shortcut creator function, for the sake of unit tests std::shared_ptr create_mesh_free_grids_manager (const ekat::Comm& comm, const int num_local_elems, const int num_gp, const int num_vertical_levels, diff --git a/components/eamxx/src/share/grid/point_grid.cpp b/components/eamxx/src/share/grid/point_grid.cpp index 4dcb6d01f0d9..fb3236a4a01c 100644 --- a/components/eamxx/src/share/grid/point_grid.cpp +++ b/components/eamxx/src/share/grid/point_grid.cpp @@ -22,6 +22,23 @@ PointGrid (const std::string& grid_name, lid2idx.sync_to_dev(); } +PointGrid:: +PointGrid (const std::string& grid_name, + const int num_my_cols, + const int num_global_cols, + const int num_vertical_levels, + const ekat::Comm& comm) + : AbstractGrid(grid_name,GridType::Point,num_my_cols,num_global_cols,num_vertical_levels,comm) +{ + create_dof_fields (get_2d_scalar_layout().rank()); + + // The lid->idx map is the identity map. + auto lid2idx = get_lid_to_idx_map(); + auto h_lid_to_idx = lid2idx.get_view(); + std::iota(h_lid_to_idx.data(),h_lid_to_idx.data()+get_num_local_dofs(),0); + lid2idx.sync_to_dev(); +} + FieldLayout PointGrid::get_2d_scalar_layout () const { diff --git a/components/eamxx/src/share/grid/point_grid.hpp b/components/eamxx/src/share/grid/point_grid.hpp index d2dede39663a..3a3d426d1cf9 100644 --- a/components/eamxx/src/share/grid/point_grid.hpp +++ b/components/eamxx/src/share/grid/point_grid.hpp @@ -32,6 +32,12 @@ class PointGrid : public AbstractGrid const int num_vertical_levels, const ekat::Comm& comm); + PointGrid (const std::string& grid_name, + const int num_my_cols, + const int num_global_cols, + const int num_vertical_levels, + const ekat::Comm& comm); + virtual ~PointGrid () = default; diff --git a/components/eamxx/src/share/grid/remap/abstract_remapper.cpp b/components/eamxx/src/share/grid/remap/abstract_remapper.cpp index 84a81a144478..6d25787794fa 100644 --- a/components/eamxx/src/share/grid/remap/abstract_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/abstract_remapper.cpp @@ -13,8 +13,8 @@ AbstractRemapper (const grid_ptr_type& src_grid, void AbstractRemapper:: registration_begins () { EKAT_REQUIRE_MSG(m_state==RepoState::Clean, - "Error! Cannot start registration on a non-clean repo.\n" - " Did you call 'registration_begins' already?\n"); + "Error! Cannot start registration on a non-clean repo.\n" + " Did you call 'registration_begins' already?\n"); do_registration_begins(); @@ -24,19 +24,27 @@ registration_begins () { void AbstractRemapper:: register_field (const identifier_type& src, const identifier_type& tgt) { EKAT_REQUIRE_MSG(m_state!=RepoState::Clean, - "Error! Cannot register fields in the remapper at this time.\n" - " Did you forget to call 'registration_begins' ?"); + "Error! Cannot register fields in the remapper at this time.\n" + " Did you forget to call 'registration_begins' ?"); EKAT_REQUIRE_MSG(m_state!=RepoState::Closed, - "Error! Cannot register fields in the remapper at this time.\n" - " Did you accidentally call 'registration_ends' already?"); - - EKAT_REQUIRE_MSG(src.get_grid_name()==m_src_grid->name(), - "Error! Source field stores the wrong grid.\n"); - EKAT_REQUIRE_MSG(tgt.get_grid_name()==m_tgt_grid->name(), - "Error! Target field stores the wrong grid.\n"); + "Error! Cannot register fields in the remapper at this time.\n" + " Did you accidentally call 'registration_ends' already?"); + + EKAT_REQUIRE_MSG(is_valid_src_layout(src.get_layout()), + "Error! Source field has an invalid layout.\n" + " - field name : " + src.name() + "\n" + " - field layout: " + to_string(src.get_layout()) + "\n"); + EKAT_REQUIRE_MSG(is_valid_tgt_layout(tgt.get_layout()), + "Error! Source field has an invalid layout.\n" + " - field name : " + tgt.name() + "\n" + " - field layout: " + to_string(tgt.get_layout()) + "\n"); EKAT_REQUIRE_MSG(compatible_layouts(src.get_layout(),tgt.get_layout()), - "Error! Source and target layouts are not compatible.\n"); + "Error! Source and target layouts are not compatible.\n" + " - src name: " + src.name() + "\n" + " - tgt name: " + tgt.name() + "\n" + " - src layout: " + to_string(src.get_layout()) + "\n" + " - tgt layout: " + to_string(tgt.get_layout()) + "\n"); do_register_field (src,tgt); @@ -54,8 +62,8 @@ register_field (const field_type& src, const field_type& tgt) { void AbstractRemapper:: bind_field (const field_type& src, const field_type& tgt) { EKAT_REQUIRE_MSG(m_state!=RepoState::Clean, - "Error! Cannot bind fields in the remapper at this time.\n" - " Did you forget to call 'registration_begins' ?"); + "Error! Cannot bind fields in the remapper at this time.\n" + " Did you forget to call 'registration_begins' ?"); const auto& src_fid = src.get_header().get_identifier(); const auto& tgt_fid = tgt.get_header().get_identifier(); @@ -63,16 +71,16 @@ bind_field (const field_type& src, const field_type& tgt) { // Try to locate the pair of fields const int ifield = find_field(src_fid, tgt_fid); EKAT_REQUIRE_MSG(ifield>=0, - "Error! The src/tgt field pair\n" - " " + src_fid.get_id_string() + "\n" - " " + tgt_fid.get_id_string() + "\n" - " was not registered. Please, register fields before binding them.\n"); + "Error! The src/tgt field pair\n" + " " + src_fid.get_id_string() + "\n" + " " + tgt_fid.get_id_string() + "\n" + " was not registered. Please, register fields before binding them.\n"); EKAT_REQUIRE_MSG(src.is_allocated(), "Error! Source field is not yet allocated.\n"); EKAT_REQUIRE_MSG(tgt.is_allocated(), "Error! Target field is not yet allocated.\n"); EKAT_REQUIRE_MSG(!m_fields_are_bound[ifield], - "Error! Field " + src_fid.get_id_string() + " already bound.\n"); + "Error! Field " + src_fid.get_id_string() + " already bound.\n"); do_bind_field(ifield,src,tgt); @@ -91,8 +99,8 @@ bind_field (const field_type& src, const field_type& tgt) { void AbstractRemapper:: registration_ends () { EKAT_REQUIRE_MSG(m_state!=RepoState::Closed, - "Error! Cannot call registration_ends at this time.\n" - " Did you accidentally call 'registration_ends' already?"); + "Error! Cannot call registration_ends at this time.\n" + " Did you accidentally call 'registration_ends' already?"); m_num_fields = m_num_registered_fields; @@ -103,25 +111,25 @@ registration_ends () { void AbstractRemapper::remap (const bool forward) { EKAT_REQUIRE_MSG(m_state!=RepoState::Open, - "Error! Cannot perform remapping at this time.\n" - " Did you forget to call 'registration_ends'?\n"); + "Error! Cannot perform remapping at this time.\n" + " Did you forget to call 'registration_ends'?\n"); EKAT_REQUIRE_MSG(m_num_bound_fields==m_num_fields, - "Error! Not all fields have been set in the remapper.\n" - " In particular, field " + + "Error! Not all fields have been set in the remapper.\n" + " In particular, field " + std::to_string(std::distance(m_fields_are_bound.begin(),std::find(m_fields_are_bound.begin(),m_fields_are_bound.end(),false))) + - " has not been bound.\n"); + " has not been bound.\n"); if (m_state!=RepoState::Clean) { if (forward) { EKAT_REQUIRE_MSG (m_fwd_allowed, - "Error! Forward remap is not allowed by this remapper.\n" - " This means that some fields on the target grid are read-only.\n"); + "Error! Forward remap is not allowed by this remapper.\n" + " This means that some fields on the target grid are read-only.\n"); do_remap_fwd (); } else { EKAT_REQUIRE_MSG (m_bwd_allowed, - "Error! Backward remap is not allowed by this remapper.\n" - " This means that some fields on the source grid are read-only.\n"); + "Error! Backward remap is not allowed by this remapper.\n" + " This means that some fields on the source grid are read-only.\n"); do_remap_bwd (); } } diff --git a/components/eamxx/src/share/grid/remap/abstract_remapper.hpp b/components/eamxx/src/share/grid/remap/abstract_remapper.hpp index 19a13110c002..afbe63b04a84 100644 --- a/components/eamxx/src/share/grid/remap/abstract_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/abstract_remapper.hpp @@ -77,7 +77,7 @@ class AbstractRemapper // during field registration). const identifier_type& get_src_field_id (const int ifield) const { EKAT_REQUIRE_MSG(ifield>=0 && ifield=0 && ifield=0 && ifield=0 && ifieldname(), - "Error! Input FieldIdentifier has the wrong grid name:\n" - " - input tgt fid grid name: " + tgt_fid.get_grid_name() + "\n" - " - remapper tgt grid name: " + m_tgt_grid->name() + "\n"); - const auto& name = tgt_fid.name(); const auto& layout = create_src_layout(tgt_fid.get_layout()); const auto& units = tgt_fid.get_units(); @@ -124,11 +119,6 @@ class AbstractRemapper } FieldIdentifier create_tgt_fid (const FieldIdentifier& src_fid) const { - EKAT_REQUIRE_MSG (src_fid.get_grid_name()==m_src_grid->name(), - "Error! Input FieldIdentifier has the wrong grid name:\n" - " - input src fid grid name: " + src_fid.get_grid_name() + "\n" - " - remapper src grid name: " + m_src_grid->name() + "\n"); - const auto& name = src_fid.name(); const auto& layout = create_tgt_layout(src_fid.get_layout()); const auto& units = src_fid.get_units(); @@ -174,6 +164,13 @@ class AbstractRemapper return src==tgt; } + virtual bool is_valid_src_layout (const layout_type& layout) const { + return m_src_grid->is_valid_layout(layout); + } + virtual bool is_valid_tgt_layout (const layout_type& layout) const { + return m_tgt_grid->is_valid_layout(layout); + } + protected: void set_grids (const grid_ptr_type& src_grid, diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index 737dcc71f718..c05ea30efdcc 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -1,6 +1,7 @@ #include "coarsening_remapper.hpp" #include "share/grid/point_grid.hpp" +#include "share/grid/grid_import_export.hpp" #include "share/io/scorpio_input.hpp" #include @@ -15,155 +16,11 @@ CoarseningRemapper:: CoarseningRemapper (const grid_ptr_type& src_grid, const std::string& map_file, const bool track_mask) - : AbstractRemapper() - , m_comm (src_grid->get_comm()) + : HorizInterpRemapperBase (src_grid,map_file,InterpType::Coarsen) , m_track_mask (track_mask) { using namespace ShortFieldTagsNames; - // Sanity checks - EKAT_REQUIRE_MSG (src_grid->type()==GridType::Point, - "Error! CoarseningRemapper only works on PointGrid grids.\n" - " - src grid name: " + src_grid->name() + "\n" - " - src_grid_type: " + e2str(src_grid->type()) + "\n"); - EKAT_REQUIRE_MSG (src_grid->is_unique(), - "Error! CoarseningRemapper requires a unique source grid.\n"); - - // This is a coarsening remapper. We only go in one direction - m_bwd_allowed = false; - - // Create io_grid, containing the indices of the triplets - // in the map file that this rank has to read - auto my_gids = get_my_triplets_gids (map_file,src_grid); - auto io_grid = std::make_shared("",my_gids.size(),0,m_comm); - - auto dofs_gids = io_grid->get_dofs_gids(); - auto dofs_gids_h = dofs_gids.get_view(); - std::memcpy(dofs_gids_h.data(),my_gids.data(),my_gids.size()*sizeof(gid_t)); - dofs_gids.sync_to_dev(); - - // Create CRS matrix views - - // Read in triplets. - const int nlweights = io_grid->get_num_local_dofs(); - std::vector row_gids_h(nlweights); - std::vector col_gids_h(nlweights); - std::vector S_h (nlweights); - - // scream's gids are of type int, while scorpio wants long int as offsets. - std::vector dofs_offsets(nlweights); - for (int i=0; i::max(); // Really big INT - for (int id=0; idget_global_min_dof_gid(); - for (int ii=0; ii ov_tgt_gids; - for (int i=0; i("ov_tgt_grid",num_ov_tgt_gids,0,m_comm); - auto ov_tgt_gids_h = ov_tgt_grid->get_dofs_gids().get_view(); - auto it = ov_tgt_gids.begin(); - for (int i=0; iget_dofs_gids().sync_to_dev(); - - m_ov_tgt_grid = ov_tgt_grid; - const int num_ov_row_gids = m_ov_tgt_grid->get_num_local_dofs(); - - // Now we have to create the weights CRS matrix - m_row_offsets = view_1d("",num_ov_row_gids+1); - m_col_lids = view_1d("",nlweights); - m_weights = view_1d("",nlweights); - - // Sort col_gids_h and row_gids_h by row gid. It is easier to sort - // the array [0,...,n), and use it later to index the row/col/weight - // views in the correct order. - std::vector id (nlweights); - std::iota(id.begin(),id.end(),0); - auto compare = [&] (const int i, const int j) -> bool { - return row_gids_h[i] < row_gids_h[j]; - }; - std::sort(id.begin(),id.end(),compare); - - // Create mirror views - auto row_offsets_h = Kokkos::create_mirror_view(m_row_offsets); - auto col_lids_h = Kokkos::create_mirror_view(m_col_lids); - auto weights_h = Kokkos::create_mirror_view(m_weights); - - for (int i=0; i row_counts(num_ov_row_gids); - for (int i=0; iget_num_vertical_levels(); - - auto tgt_grid_gids = m_ov_tgt_grid->get_unique_gids (); - const int ngids = tgt_grid_gids.size(); - - auto tgt_grid = std::make_shared("horiz_remap_tgt_grid",ngids,nlevs,m_comm); - - auto tgt_grid_gids_h = tgt_grid->get_dofs_gids().get_view(); - std::memcpy(tgt_grid_gids_h.data(),tgt_grid_gids.data(),ngids*sizeof(gid_t)); - tgt_grid->get_dofs_gids().sync_to_dev(); - - this->set_grids(src_grid,tgt_grid); - // Replicate the src grid geo data in the tgt grid. We use this remapper to do // the remapping (if needed), and clean it up afterwards. const auto& src_geo_data_names = src_grid->get_geometry_data_names(); @@ -176,12 +33,12 @@ CoarseningRemapper (const grid_ptr_type& src_grid, // Not a field to be coarsened (perhaps a vertical coordinate field). // Simply copy it in the tgt grid, but we still need to assign the new grid name. FieldIdentifier tgt_data_fid(src_data_fid.name(),src_data_fid.get_layout(),src_data_fid.get_units(),m_tgt_grid->name()); - auto tgt_data = tgt_grid->create_geometry_data(tgt_data_fid); + auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); tgt_data.deep_copy(src_data); } else { // This field needs to be remapped auto tgt_data_fid = create_tgt_fid(src_data_fid); - auto tgt_data = tgt_grid->create_geometry_data(tgt_data_fid); + auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); register_field(src_data,tgt_data); } } @@ -189,14 +46,14 @@ CoarseningRemapper (const grid_ptr_type& src_grid, if (get_num_fields()>0) { remap(true); - // The remap phase only alters the fields on device. We need to sync them to host as well + // The remap phase only alters the fields on device. + // We need to sync them to host as well for (int i=0; iget_2d_scalar_layout(); - break; - case LayoutType::Vector2D: - src = m_src_grid->get_2d_vector_layout(CMP,vec_dim); - break; - case LayoutType::Scalar3D: - src = m_src_grid->get_3d_scalar_layout(midpoints); - break; - case LayoutType::Vector3D: - src = m_src_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); - break; - default: - EKAT_ERROR_MSG ("Layout not supported by CoarseningRemapper: " + e2str(lt) + "\n"); - } - return src; -} -FieldLayout CoarseningRemapper:: -create_tgt_layout (const FieldLayout& src_layout) const -{ - using namespace ShortFieldTagsNames; - const auto lt = get_layout_type(src_layout.tags()); - auto tgt = FieldLayout::invalid(); - const bool midpoints = src_layout.has_tag(LEV); - const int vec_dim = src_layout.is_vector_layout() ? src_layout.dim(CMP) : -1; - switch (lt) { - case LayoutType::Scalar2D: - tgt = m_tgt_grid->get_2d_scalar_layout(); - break; - case LayoutType::Vector2D: - tgt = m_tgt_grid->get_2d_vector_layout(CMP,vec_dim); - break; - case LayoutType::Scalar3D: - tgt = m_tgt_grid->get_3d_scalar_layout(midpoints); - break; - case LayoutType::Vector3D: - tgt = m_tgt_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); - break; - default: - EKAT_ERROR_MSG ("Layout not supported by CoarseningRemapper: " + e2str(lt) + "\n"); - } - return tgt; -} - -void CoarseningRemapper:: -do_register_field (const identifier_type& src, const identifier_type& tgt) -{ - m_src_fields.push_back(field_type(src)); - m_tgt_fields.push_back(field_type(tgt)); -} - void CoarseningRemapper:: do_bind_field (const int ifield, const field_type& src, const field_type& tgt) { - EKAT_REQUIRE_MSG ( - src.get_header().get_identifier().get_layout().rank()>1 || - src.get_header().get_alloc_properties().get_padding()==0, - "Error! We don't support 2d scalar fields that are padded.\n"); - EKAT_REQUIRE_MSG ( - tgt.get_header().get_identifier().get_layout().rank()>1 || - tgt.get_header().get_alloc_properties().get_padding()==0, - "Error! We don't support 2d scalar fields that are padded.\n"); - m_src_fields[ifield] = src; - m_tgt_fields[ifield] = tgt; - // Assume no mask tracking for this field. Can correct below m_field_idx_to_mask_idx[ifield] = -1; if (m_track_mask) { - const auto& src_extra = src.get_header().get_extra_data(); - if (src_extra.count("mask_data")==1) { + if (src.get_header().has_extra_data("mask_data")) { // First, check that we also have the mask value, to be used if mask_data is too small - EKAT_REQUIRE_MSG (src_extra.count("mask_value")==1, + EKAT_REQUIRE_MSG (src.get_header().has_extra_data("mask_value"), "Error! Field " + src.name() + " stores a mask field but not a mask value.\n"); - const auto& src_mask_val = src_extra.at("mask_value"); + const auto& src_mask_val = src.get_header().get_extra_data("mask_value"); + + Field tgt_copy = tgt; - auto& tgt_hdr = m_tgt_fields[ifield].get_header(); - if (tgt_hdr.get_extra_data().count("mask_value")==1) { - const auto& tgt_mask_val = tgt_hdr.get_extra_data().at("mask_value"); + auto& tgt_hdr = tgt_copy.get_header(); + if (tgt_hdr.has_extra_data("mask_value")) { + const auto& tgt_mask_val = tgt_hdr.get_extra_data("mask_value"); - EKAT_REQUIRE_MSG (ekat::any_cast(tgt_mask_val)==ekat::any_cast(src_mask_val), + EKAT_REQUIRE_MSG (tgt_mask_val==src_mask_val, "Error! Target field stores a mask data different from the src field.\n" " - src field name: " + src.name() + "\n" " - tgt field name: " + tgt.name() + "\n" - " - src mask value: " << ekat::any_cast(src_mask_val) << "\n" - " - tgt mask value: " << ekat::any_cast(tgt_mask_val) << "\n"); + " - src mask value: " << src_mask_val << "\n" + " - tgt mask value: " << tgt_mask_val << "\n"); } else { tgt_hdr.set_extra_data("mask_value",src_mask_val); } // Then, register the mask field, if not yet registered - const auto& src_mask = ekat::any_cast(src_extra.at("mask_data")); + const auto& src_mask = src.get_header().get_extra_data("mask_data"); const auto& src_mask_fid = src_mask.get_header().get_identifier(); // Make sure fields representing masks are not themselves meant to be masked. - EKAT_REQUIRE_MSG(src_mask.get_header().get_extra_data().count("mask_data")==0, + EKAT_REQUIRE_MSG(not src_mask.get_header().has_extra_data("mask_data"), "Error! A mask field cannot be itself masked.\n" " - field name: " + src.name() + "\n" " - mask field name: " + src_mask.name() + "\n"); @@ -347,21 +134,7 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) " - mask layout: " + to_string(m_lt) + "\n"); } } - - // If this was the last field to be bound, we can setup the MPI schedule - if (this->m_state==RepoState::Closed && - (this->m_num_bound_fields+1)==this->m_num_registered_fields) { - create_ov_tgt_fields (); - setup_mpi_data_structures (); - } -} - -void CoarseningRemapper::do_registration_ends () -{ - if (this->m_num_bound_fields==this->m_num_registered_fields) { - create_ov_tgt_fields (); - setup_mpi_data_structures (); - } + HorizInterpRemapperBase::do_bind_field(ifield,src,tgt); } void CoarseningRemapper::do_remap_fwd () @@ -377,36 +150,36 @@ void CoarseningRemapper::do_remap_fwd () // TODO: Add check that if there are mask values they are either 1's or 0's for unmasked/masked. + // Helpef function, to establish if a field can be handled with packs + auto can_pack_field = [](const Field& f) { + const auto& ap = f.get_header().get_alloc_properties(); + return (ap.get_last_extent() % SCREAM_PACK_SIZE) == 0; + }; + // Loop over each field - constexpr auto can_pack = SCREAM_PACK_SIZE>1; for (int i=0; i0) { // Pass the mask to the local_mat_vec routine - auto mask = m_src_fields[mask_idx]; - // Dispatch kernel with the largest possible pack size - const auto& src_ap = f_src.get_header().get_alloc_properties(); - const auto& ov_tgt_ap = f_ov_tgt.get_header().get_alloc_properties(); - if (can_pack && src_ap.is_compatible>() && - ov_tgt_ap.is_compatible>()) { - local_mat_vec(f_src,f_ov_tgt,&mask); + const auto& mask = m_src_fields[mask_idx]; + + // If possible, dispatch kernel with SCREAM_PACK_SIZE + if (can_pack_field(f_src) and can_pack_field(f_ov)) { + local_mat_vec(f_src,f_ov,mask); } else { - local_mat_vec<1>(f_src,f_ov_tgt,&mask); + local_mat_vec<1>(f_src,f_ov,mask); } } else { - // Dispatch kernel with the largest possible pack size - const auto& src_ap = f_src.get_header().get_alloc_properties(); - const auto& ov_tgt_ap = f_ov_tgt.get_header().get_alloc_properties(); - if (can_pack && src_ap.is_compatible>() && - ov_tgt_ap.is_compatible>()) { - local_mat_vec(f_src,f_ov_tgt); + // If possible, dispatch kernel with SCREAM_PACK_SIZE + if (can_pack_field(f_src) and can_pack_field(f_ov)) { + local_mat_vec(f_src,f_ov); } else { - local_mat_vec<1>(f_src,f_ov_tgt); + local_mat_vec<1>(f_src,f_ov); } } } @@ -433,8 +206,7 @@ void CoarseningRemapper::do_remap_fwd () if (mask_idx>0) { // Then this field did use a mask const auto& mask = m_tgt_fields[mask_idx]; - const auto& tgt_ap = f_tgt.get_header().get_alloc_properties(); - if (can_pack && tgt_ap.is_compatible>()) { + if (can_pack_field(f_tgt)) { rescale_masked_fields(f_tgt,mask); } else { rescale_masked_fields<1>(f_tgt,mask); @@ -457,19 +229,22 @@ rescale_masked_fields (const Field& x, const Field& mask) const const auto& layout = x.get_header().get_identifier().get_layout(); const int rank = layout.rank(); const int ncols = m_tgt_grid->get_num_local_dofs(); - const auto x_extra = x.get_header().get_extra_data(); Real mask_val = std::numeric_limits::max()/10.0; - if (x_extra.count("mask_value")) { - mask_val = ekat::any_cast(x_extra.at("mask_value")); + if (x.get_header().has_extra_data("mask_value")) { + mask_val = x.get_header().get_extra_data("mask_value"); } else { EKAT_ERROR_MSG ("ERROR! Field " + x.name() + " is masked, but stores no mask_value extra data.\n"); } const Real mask_threshold = std::numeric_limits::epsilon(); // TODO: Should we not hardcode the threshold for simply masking out the column. + switch (rank) { case 1: { - auto x_view = x.get_view< Real*>(); - auto m_view = mask.get_view(); + // Unlike get_view, get_strided_view returns a LayoutStride view, + // therefore allowing the 1d field to be a subfield of a 2d field + // along the 2nd dimension. + auto x_view = x.get_strided_view< Real*>(); + auto m_view = mask.get_strided_view(); Kokkos::parallel_for(RangePolicy(0,ncols), KOKKOS_LAMBDA(const int& icol) { if (m_view(icol)>mask_threshold) { @@ -483,60 +258,98 @@ rescale_masked_fields (const Field& x, const Field& mask) const case 2: { auto x_view = x.get_view< Pack**>(); - auto m_view = mask.get_view(); + bool mask1d = mask.rank()==1; + view_1d mask_1d; + view_2d mask_2d; + // If the mask comes from FieldAtLevel, it's only defined on columns (rank=1) + // If the mask comes from vert interpolation remapper, it is defined on ncols x nlevs (rank=2) + if (mask.rank()==1) { + mask_1d = mask.get_view(); + } else { + mask_2d = mask.get_view(); + } const int dim1 = PackInfo::num_packs(layout.dim(1)); auto policy = ESU::get_default_team_policy(ncols,dim1); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const auto icol = team.league_rank(); auto x_sub = ekat::subview(x_view,icol); - auto m_sub = ekat::subview(m_view,icol); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1), - [&](const int j){ - auto masked = m_sub(j) > mask_threshold; - if (masked.any()) { - x_sub(j).set(masked,x_sub(j)/m_sub(j)); + if (mask1d) { + auto mask = mask_1d(icol); + if (mask>mask_threshold) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1), + [&](const int j){ + x_sub(j) /= mask; + }); } - x_sub(j).set(!masked,mask_val); - }); + } else { + auto m_sub = ekat::subview(mask_2d,icol); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1), + [&](const int j){ + auto masked = m_sub(j) > mask_threshold; + if (masked.any()) { + x_sub(j).set(masked,x_sub(j)/m_sub(j)); + } + x_sub(j).set(!masked,mask_val); + }); + } }); break; } case 3: { auto x_view = x.get_view< Pack***>(); - auto m_view = mask.get_view(); + bool mask1d = mask.rank()==1; + view_1d mask_1d; + view_2d mask_2d; + // If the mask comes from FieldAtLevel, it's only defined on columns (rank=1) + // If the mask comes from vert interpolation remapper, it is defined on ncols x nlevs (rank=2) + if (mask.rank()==1) { + mask_1d = mask.get_view(); + } else { + mask_2d = mask.get_view(); + } const int dim1 = layout.dim(1); const int dim2 = PackInfo::num_packs(layout.dim(2)); auto policy = ESU::get_default_team_policy(ncols,dim1*dim2); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const auto icol = team.league_rank(); - auto m_sub = ekat::subview(m_view,icol); - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2), - [&](const int idx){ - const int j = idx / dim2; - const int k = idx % dim2; - auto x_sub = ekat::subview(x_view,icol,j); - auto masked = m_sub(k) > mask_threshold; - - if (masked.any()) { - x_sub(k).set(masked,x_sub(k)/m_sub(k)); + if (mask1d) { + auto mask = mask_1d(icol); + if (mask>mask_threshold) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2), + [&](const int idx){ + const int j = idx / dim2; + const int k = idx % dim2; + auto x_sub = ekat::subview(x_view,icol,j); + x_sub(k) /= mask; + }); } - x_sub(k).set(!masked,mask_val); - }); + } else { + auto m_sub = ekat::subview(mask_2d,icol); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2), + [&](const int idx){ + const int j = idx / dim2; + const int k = idx % dim2; + auto x_sub = ekat::subview(x_view,icol,j); + auto masked = m_sub(k) > mask_threshold; + + if (masked.any()) { + x_sub(k).set(masked,x_sub(k)/m_sub(k)); + } + x_sub(k).set(!masked,mask_val); + }); + } }); break; } } - - } template void CoarseningRemapper:: -local_mat_vec (const Field& x, const Field& y, const Field* mask) const +local_mat_vec (const Field& x, const Field& y, const Field& mask) const { using RangePolicy = typename KT::RangePolicy; using MemberType = typename KT::MemberType; @@ -546,7 +359,7 @@ local_mat_vec (const Field& x, const Field& y, const Field* mask) const const auto& src_layout = x.get_header().get_identifier().get_layout(); const int rank = src_layout.rank(); - const int nrows = m_ov_tgt_grid->get_num_local_dofs(); + const int nrows = m_ov_coarse_grid->get_num_local_dofs(); auto row_offsets = m_row_offsets; auto col_lids = m_col_lids; auto weights = m_weights; @@ -556,26 +369,19 @@ local_mat_vec (const Field& x, const Field& y, const Field* mask) const // loop to zero out y before the mat-vec. case 1: { - auto x_view = x.get_view(); - auto y_view = y.get_view< Real*>(); - view_1d mask_view; - if (mask != nullptr) { - mask_view = mask->get_view(); - } + // Unlike get_view, get_strided_view returns a LayoutStride view, + // therefore allowing the 1d field to be a subfield of a 2d field + // along the 2nd dimension. + auto x_view = x.get_strided_view(); + auto y_view = y.get_strided_view< Real*>(); + auto mask_view = mask.get_strided_view(); Kokkos::parallel_for(RangePolicy(0,nrows), KOKKOS_LAMBDA(const int& row) { const auto beg = row_offsets(row); const auto end = row_offsets(row+1); - if (mask != nullptr) { - y_view(row) = weights(beg)*x_view(col_lids(beg))*mask_view(col_lids(beg)); - for (int icol=beg+1; icol(); auto y_view = y.get_view< Pack**>(); - view_2d mask_view; - if (mask != nullptr) { - mask_view = mask->get_view(); + view_1d mask_1d; + view_2d mask_2d; + // If the mask comes from FieldAtLevel, it's only defined on columns (rank=1) + // If the mask comes from vert interpolation remapper, it is defined on ncols x nlevs (rank=2) + bool mask1d = mask.rank()==1; + if (mask1d) { + mask_1d = mask.get_view(); + } else { + mask_2d = mask.get_view(); } const int dim1 = PackInfo::num_packs(src_layout.dim(1)); auto policy = ESU::get_default_team_policy(nrows,dim1); @@ -598,16 +410,11 @@ local_mat_vec (const Field& x, const Field& y, const Field* mask) const const auto end = row_offsets(row+1); Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1), [&](const int j){ - if (mask != nullptr) { - y_view(row,j) = weights(beg)*x_view(col_lids(beg),j)*mask_view(col_lids(beg),j); - for (int icol=beg+1; icol(); auto y_view = y.get_view< Pack***>(); // Note, the mask is still assumed to be defined on COLxLEV so still only 2D for case 3. - view_2d mask_view; - if (mask != nullptr) { - mask_view = mask->get_view(); + view_1d mask_1d; + view_2d mask_2d; + bool mask1d = mask.rank()==1; + // If the mask comes from FieldAtLevel, it's only defined on columns (rank=1) + // If the mask comes from vert interpolation remapper, it is defined on ncols x nlevs (rank=2) + if (mask1d) { + mask_1d = mask.get_view(); + } else { + mask_2d = mask.get_view(); } const int dim1 = src_layout.dim(1); const int dim2 = PackInfo::num_packs(src_layout.dim(2)); @@ -635,16 +448,11 @@ local_mat_vec (const Field& x, const Field& y, const Field* mask) const [&](const int idx){ const int j = idx / dim2; const int k = idx % dim2; - if (mask != nullptr) { - y_view(row,j,k) = weights(beg)*x_view(col_lids(beg),j,k)*mask_view(col_lids(beg),k); - for (int icol=beg+1; icol; - const int num_send_gids = m_ov_tgt_grid->get_num_local_dofs(); + const int num_send_gids = m_ov_coarse_grid->get_num_local_dofs(); const auto pid_lid_start = m_send_pid_lids_start; const auto lids_pids = m_send_lids_pids; const auto buf = m_send_buffer; for (int ifield=0; ifield(); + // Unlike get_view, get_strided_view returns a LayoutStride view, + // therefore allowing the 1d field to be a subfield of a 2d field + // along the 2nd dimension. + auto v = f.get_strided_view(); Kokkos::parallel_for(RangePolicy(0,num_send_gids), KOKKOS_LAMBDA(const int& i){ const int lid = lids_pids(i,0); @@ -688,11 +498,11 @@ void CoarseningRemapper::pack_and_send () buf (offset + lidpos) = v(lid); }); } break; - case LayoutType::Vector2D: + case 2: { auto v = f.get_view(); - const int ndims = fl.dim(1); - auto policy = ESU::get_default_team_policy(num_send_gids,ndims); + const int dim1 = fl.dim(1); + auto policy = ESU::get_default_team_policy(num_send_gids,dim1); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team){ const int i = team.league_rank(); @@ -701,37 +511,18 @@ void CoarseningRemapper::pack_and_send () const int lidpos = i - pid_lid_start(pid); const int offset = f_pid_offsets(pid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team,ndims), + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1), [&](const int idim) { - buf(offset + lidpos*ndims + idim) = v(lid,idim); - }); - }); - } break; - case LayoutType::Scalar3D: - { - auto v = f.get_view(); - const int nlevs = fl.dims().back(); - auto policy = ESU::get_default_team_policy(num_send_gids,nlevs); - Kokkos::parallel_for(policy, - KOKKOS_LAMBDA(const MemberType& team){ - const int i = team.league_rank(); - const int lid = lids_pids(i,0); - const int pid = lids_pids(i,1); - const int lidpos = i - pid_lid_start(pid); - const int offset = f_pid_offsets(pid); - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team,nlevs), - [&](const int ilev) { - buf(offset + lidpos*nlevs + ilev) = v(lid,ilev); + buf(offset + lidpos*dim1 + idim) = v(lid,idim); }); }); } break; - case LayoutType::Vector3D: + case 3: { auto v = f.get_view(); - const int ndims = fl.dim(1); - const int nlevs = fl.dims().back(); - auto policy = ESU::get_default_team_policy(num_send_gids,ndims*nlevs); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + auto policy = ESU::get_default_team_policy(num_send_gids,dim1*dim2); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team){ const int i = team.league_rank(); @@ -740,11 +531,11 @@ void CoarseningRemapper::pack_and_send () const int lidpos = i - pid_lid_start(pid); const int offset = f_pid_offsets(pid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team,ndims*nlevs), + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2), [&](const int idx) { - const int idim = idx / nlevs; - const int ilev = idx % nlevs; - buf(offset + lidpos*ndims*nlevs + idim*nlevs + ilev) = v(lid,idim,ilev); + const int idim = idx / dim2; + const int ilev = idx % dim2; + buf(offset + lidpos*dim1*dim2 + idim*dim2 + ilev) = v(lid,idim,ilev); }); }); } break; @@ -752,10 +543,14 @@ void CoarseningRemapper::pack_and_send () default: EKAT_ERROR_MSG ("Unexpected field rank in CoarseningRemapper::pack.\n" " - MPI rank : " + std::to_string(m_comm.rank()) + "\n" + " - field name: " + f.name() + "\n" " - field rank: " + std::to_string(fl.rank()) + "\n"); } } + // Ensure all threads are done packing before firing off the sends + Kokkos::fence(); + // If MPI does not use dev pointers, we need to deep copy from dev to host if (not MpiOnDev) { Kokkos::deep_copy (m_mpi_send_buffer,m_send_buffer); @@ -795,15 +590,16 @@ void CoarseningRemapper::recv_and_unpack () for (int ifield=0; ifield(); + // Unlike get_view, get_strided_view returns a LayoutStride view, + // therefore allowing the 1d field to be a subfield of a 2d field + // along the 2nd dimension. + auto v = f.get_strided_view(); Kokkos::parallel_for(RangePolicy(0,num_tgt_dofs), KOKKOS_LAMBDA(const int& lid){ const int recv_beg = recv_lids_beg(lid); @@ -816,11 +612,11 @@ void CoarseningRemapper::recv_and_unpack () } }); } break; - case LayoutType::Vector2D: + case 2: { auto v = f.get_view(); - const int ndims = fl.dim(1); - auto policy = ESU::get_default_team_policy(num_tgt_dofs,ndims); + const int dim1 = fl.dim(1); + auto policy = ESU::get_default_team_policy(num_tgt_dofs,dim1); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team){ const int lid = team.league_rank(); @@ -829,42 +625,20 @@ void CoarseningRemapper::recv_and_unpack () for (int irecv=recv_beg; irecv(); - const int nlevs = fl.dims().back(); - auto policy = ESU::get_default_team_policy(num_tgt_dofs,nlevs); - Kokkos::parallel_for(policy, - KOKKOS_LAMBDA(const MemberType& team){ - const int lid = team.league_rank(); - const int recv_beg = recv_lids_beg(lid); - const int recv_end = recv_lids_end(lid); - for (int irecv=recv_beg; irecv(); - const int ndims = fl.dim(1); - const int nlevs = fl.dims().back(); - auto policy = ESU::get_default_team_policy(num_tgt_dofs,nlevs*ndims); + const int dim1 = fl.dim(1); + const int dim2 = fl.dims().back(); + auto policy = ESU::get_default_team_policy(num_tgt_dofs,dim2*dim1); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team){ const int lid = team.league_rank(); @@ -873,13 +647,13 @@ void CoarseningRemapper::recv_and_unpack () for (int irecv=recv_beg; irecv -CoarseningRemapper:: -get_my_triplets_gids (const std::string& map_file, - const grid_ptr_type& src_grid) const -{ - using namespace ShortFieldTagsNames; - - scorpio::register_file(map_file,scorpio::FileMode::Read); - // 1. Create a "helper" grid, with as many dofs as the number - // of triplets in the map file, and divided linearly across ranks - const int ngweights = scorpio::get_dimlen(map_file,"n_s"); - const auto io_grid_linear = create_point_grid ("helper",ngweights,1,m_comm); - const int nlweights = io_grid_linear->get_num_local_dofs(); - - gid_t offset = nlweights; - m_comm.scan(&offset,1,MPI_SUM); - offset -= nlweights; // scan is inclusive, but we need exclusive - - // 2. Read a chunk of triplets col indices - std::vector cols(nlweights); - std::vector rows(nlweights); // Needed to calculate min_dof - const std::string idx_decomp_tag = "coarsening_remapper::get_my_triplet_gids_int_dim" + std::to_string(nlweights); - scorpio::get_variable(map_file, "col", "col", {"n_s"}, "int", idx_decomp_tag); - scorpio::get_variable(map_file, "row", "row", {"n_s"}, "int", idx_decomp_tag); - std::vector dofs_offsets(nlweights); - std::iota(dofs_offsets.begin(),dofs_offsets.end(),offset); - scorpio::set_dof(map_file,"col",nlweights,dofs_offsets.data()); - scorpio::set_dof(map_file,"row",nlweights,dofs_offsets.data()); - scorpio::set_decomp(map_file); - scorpio::grid_read_data_array(map_file,"col",-1,cols.data(),cols.size()); - scorpio::grid_read_data_array(map_file,"row",-1,rows.data(),rows.size()); - scorpio::eam_pio_closefile(map_file); - - // Offset the cols ids to match the source grid. - // Determine the min id among the cols array, we - // also add the min_dof for the grid. - int remap_min_dof = std::numeric_limits::max(); - for (int id=0; idget_global_min_dof_gid(); - for (auto& id : cols) { - id -= col_offset; - } - - // 3. Get the owners of the cols gids we read in, according to the src grid - auto owners = src_grid->get_owners(cols); - - // 4. Group gids we read by the pid we need to send them to - std::map> pid2gids_send; - for (int i=0; i my_triplets_gids(num_my_triplets); - int num_copied = 0; - for (const auto& it : pid2gids_recv) { - auto dst = my_triplets_gids.data()+num_copied; - auto src = it.second.data(); - std::memcpy (dst,src,it.second.size()*sizeof(int)); - num_copied += it.second.size(); - } - - return my_triplets_gids; -} - std::vector CoarseningRemapper::get_pids_for_recv (const std::vector& send_to_pids) const { @@ -1066,32 +760,6 @@ recv_gids_from_pids (const std::map>& pid2gids_send) const return pid2gids_recv; } -void CoarseningRemapper::create_ov_tgt_fields () -{ - using FL = FieldLayout; - m_ov_tgt_fields.reserve(m_num_fields); - const int num_ov_cols = m_ov_tgt_grid->get_num_local_dofs(); - const auto ov_gn = m_ov_tgt_grid->name(); - for (int i=0; i field_col_size (m_num_fields); int sum_fields_col_sizes = 0; for (int i=0; i0) { - field_col_size[i] = fl.size() / fl.dim(0); - } else { - field_col_size[i] = fl.size(); - } + field_col_size[i] = fl.strip_dim(COL).size(); sum_fields_col_sizes += field_col_size[i]; } @@ -1119,15 +783,15 @@ void CoarseningRemapper::setup_mpi_data_structures () // Setup SEND structures // // --------------------------------------------------------- // - // 1. Retrieve pid (and associated lid) of all ov_tgt gids + // 1. Retrieve pid (and associated lid) of all ov gids // on the tgt grid - const auto ov_gids = m_ov_tgt_grid->get_dofs_gids().get_view(); + const auto ov_gids = m_ov_coarse_grid->get_dofs_gids().get_view(); auto gids_owners = m_tgt_grid->get_owners (ov_gids); // 2. Group dofs to send by remote pid const int num_ov_gids = ov_gids.size(); std::map> pid2lids_send; - std::map> pid2gids_send; + std::map> pid2gids_send; for (int i=0; i> lid2pids_recv(num_tgt_dofs); int num_total_recv_gids = 0; + auto tgt_gid2lid = m_tgt_grid->get_gid2lid_map(); for (const auto& it : pid2gids_recv) { const int pid = it.first; for (auto gid : it.second) { - const int lid = gid2lid(gid,m_tgt_grid); + const int lid = tgt_gid2lid[gid]; lid2pids_recv[lid].push_back(pid); } num_total_recv_gids += it.second.size(); @@ -1219,7 +884,7 @@ void CoarseningRemapper::setup_mpi_data_structures () auto recv_lids_beg_h = Kokkos::create_mirror_view(m_recv_lids_beg); auto recv_lids_end_h = Kokkos::create_mirror_view(m_recv_lids_end); - auto tgt_dofs_h = m_tgt_grid->get_dofs_gids().get_view(); + auto tgt_dofs_h = m_tgt_grid->get_dofs_gids().get_view(); for (int i=0,pos=0; i namespace scream @@ -35,15 +33,12 @@ namespace scream * An obvious future development would be to use some scratch memory * for these fields, so to not increase memory pressure. * - * The setup of the class uses a bunch of RMA mpi operations, since they - * are more convenient when ranks don't know where data is coming from - * or how much data is coming from each rank. The runtime operations, - * however, use the classic send/recv paradigm, where data is packed in - * a buffer, sent to the recv rank, and then unpacked and accumulated - * into the result. + * The setup as well as the runtime operations use classic send/recv + * MPI calls, where data is packed in a buffer and sent to the recv rank, + * where it is then unpacked and accumulated into the result. */ -class CoarseningRemapper : public AbstractRemapper +class CoarseningRemapper : public HorizInterpRemapperBase { public: @@ -53,82 +48,18 @@ class CoarseningRemapper : public AbstractRemapper ~CoarseningRemapper (); - FieldLayout create_src_layout (const FieldLayout& tgt_layout) const override; - FieldLayout create_tgt_layout (const FieldLayout& src_layout) const override; - - bool compatible_layouts (const layout_type& src, - const layout_type& tgt) const override { - // Same type of layout, and same sizes except for possibly the first one - // Note: we can't do tgt.size()/tgt.dim(0), since there may be 0 tgt gids - // on some ranks, which means tgt.dim(0)=0. - int src_col_size = 1; - for (int i=1; i; - using gid_t = AbstractGrid::gid_type; - - template - using RPack = ekat::Pack; - - template - using view_1d = typename KT::template view_1d; template using view_2d = typename KT::template view_2d; - void create_ov_tgt_fields (); - void setup_mpi_data_structures (); - - int gid2lid (const gid_t gid, const grid_ptr_type& grid) const { - const auto gids = grid->get_dofs_gids().get_view(); - const auto beg = gids.data(); - const auto end = gids.data()+grid->get_num_local_dofs(); - const auto it = std::find(beg,end,gid); - return it==end ? -1 : std::distance(beg,it); - } - - std::vector - get_my_triplets_gids (const std::string& map_file, - const grid_ptr_type& src_grid) const; + void setup_mpi_data_structures () override; std::vector get_pids_for_recv (const std::vector& send_to_pids) const; @@ -144,16 +75,15 @@ class CoarseningRemapper : public AbstractRemapper public: #endif template - void local_mat_vec (const Field& f_src, const Field& f_tgt, const Field* mask = nullptr) const; + void local_mat_vec (const Field& f_src, const Field& f_tgt, const Field& mask) const; template void rescale_masked_fields (const Field& f_tgt, const Field& f_mask) const; void pack_and_send (); void recv_and_unpack (); + // Overload, not hide + using HorizInterpRemapperBase::local_mat_vec; protected: - // If a field - - ekat::Comm m_comm; static constexpr bool MpiOnDev = SCREAM_MPI_ON_DEVICE; @@ -165,24 +95,10 @@ class CoarseningRemapper : public AbstractRemapper typename view_1d::HostMirror >::type; - // An "overlapped" tgt grid, that is a version of the tgt grid where - // ranks own all rows that are affected by local dofs in their src grid - grid_ptr_type m_ov_tgt_grid; - - // Source, target, and overlapped-target fields - std::vector m_src_fields; - std::vector m_ov_tgt_fields; - std::vector m_tgt_fields; - // Mask fields, if needed bool m_track_mask; std::map m_field_idx_to_mask_idx; - // ----- Sparse matrix CRS representation ---- // - view_1d m_row_offsets; - view_1d m_col_lids; - view_1d m_weights; - // ------- MPI data structures -------- // // The send/recv buf for pack/unpack diff --git a/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp b/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp index 52bef384e561..3bb593923795 100644 --- a/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp @@ -37,8 +37,12 @@ class DoNothingRemapper : public AbstractRemapper FieldLayout create_src_layout (const FieldLayout& tgt) const override { using namespace ShortFieldTagsNames; + EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt), + "[DoNothingRemapper] Error! Input target layout is not valid for this remapper.\n" + " - input layout: " + to_string(tgt)); + auto type = get_layout_type(tgt.tags()); - FieldLayout src = {{}}; + auto src = FieldLayout::invalid(); switch (type) { case LayoutType::Scalar2D: src = this->m_src_grid->get_2d_scalar_layout(); @@ -61,8 +65,12 @@ class DoNothingRemapper : public AbstractRemapper FieldLayout create_tgt_layout (const FieldLayout& src) const override { using namespace ShortFieldTagsNames; + EKAT_REQUIRE_MSG (is_valid_src_layout(src), + "[DoNothingRemapper] Error! Input source layout is not valid for this remapper.\n" + " - input layout: " + to_string(src)); + auto type = get_layout_type(src.tags()); - FieldLayout tgt = {{}}; + auto tgt = FieldLayout::invalid(); switch (type) { case LayoutType::Scalar2D: tgt = this->m_tgt_grid->get_2d_scalar_layout(); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp new file mode 100644 index 000000000000..cb65bdda88d6 --- /dev/null +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -0,0 +1,522 @@ +#include "horiz_interp_remapper_base.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/grid/grid_import_export.hpp" +#include "share/io/scorpio_input.hpp" + +#include +#include +#include + +namespace scream +{ + +HorizInterpRemapperBase:: +HorizInterpRemapperBase (const grid_ptr_type& fine_grid, + const std::string& map_file, + const InterpType type) + : m_fine_grid(fine_grid) + , m_type (type) + , m_comm (fine_grid->get_comm()) +{ + // Sanity checks + EKAT_REQUIRE_MSG (fine_grid->type()==GridType::Point, + "Error! Horizontal interpolatory remap only works on PointGrid grids.\n" + " - fine grid name: " + fine_grid->name() + "\n" + " - fine_grid_type: " + e2str(fine_grid->type()) + "\n"); + EKAT_REQUIRE_MSG (fine_grid->is_unique(), + "Error! CoarseningRemapper requires a unique source grid.\n"); + + // This is a special remapper. We only go in one direction + m_bwd_allowed = false; + + // Read the map file, loading the triplets this rank needs for the crs matrix + // in the map file that this rank has to read + auto my_triplets = get_my_triplets (map_file); + + // Create coarse/ov_coarse grids + create_coarse_grids (my_triplets); + + // Set src/tgt grid, based on interpolation type + if (m_type==InterpType::Refine) { + set_grids (m_coarse_grid,m_fine_grid); + } else { + set_grids (m_fine_grid,m_coarse_grid); + } + + // Create crs matrix + create_crs_matrix_structures (my_triplets); +} + +FieldLayout HorizInterpRemapperBase:: +create_src_layout (const FieldLayout& tgt_layout) const +{ + EKAT_REQUIRE_MSG (m_src_grid!=nullptr, + "Error! Cannot create source layout until the source grid has been set.\n"); + + EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), + "[HorizInterpRemapperBase] Error! Input target layout is not valid for this remapper.\n" + " - input layout: " + to_string(tgt_layout)); + + using namespace ShortFieldTagsNames; + const auto lt = get_layout_type(tgt_layout.tags()); + const bool midpoints = tgt_layout.has_tag(LEV); + const int vec_dim = tgt_layout.is_vector_layout() ? tgt_layout.dim(CMP) : -1; + auto src = FieldLayout::invalid(); + switch (lt) { + case LayoutType::Scalar2D: + src = m_src_grid->get_2d_scalar_layout(); + break; + case LayoutType::Vector2D: + src = m_src_grid->get_2d_vector_layout(CMP,vec_dim); + break; + case LayoutType::Scalar3D: + src = m_src_grid->get_3d_scalar_layout(midpoints); + break; + case LayoutType::Vector3D: + src = m_src_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); + break; + default: + EKAT_ERROR_MSG ("Layout not supported by CoarseningRemapper: " + e2str(lt) + "\n"); + } + return src; +} + +FieldLayout HorizInterpRemapperBase:: +create_tgt_layout (const FieldLayout& src_layout) const +{ + EKAT_REQUIRE_MSG (m_tgt_grid!=nullptr, + "Error! Cannot create target layout until the target grid has been set.\n"); + + EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), + "[HorizInterpRemapperBase] Error! Input source layout is not valid for this remapper.\n" + " - input layout: " + to_string(src_layout)); + + using namespace ShortFieldTagsNames; + const auto lt = get_layout_type(src_layout.tags()); + auto tgt = FieldLayout::invalid(); + const bool midpoints = src_layout.has_tag(LEV); + const int vec_dim = src_layout.is_vector_layout() ? src_layout.dim(CMP) : -1; + switch (lt) { + case LayoutType::Scalar2D: + tgt = m_tgt_grid->get_2d_scalar_layout(); + break; + case LayoutType::Vector2D: + tgt = m_tgt_grid->get_2d_vector_layout(CMP,vec_dim); + break; + case LayoutType::Scalar3D: + tgt = m_tgt_grid->get_3d_scalar_layout(midpoints); + break; + case LayoutType::Vector3D: + tgt = m_tgt_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); + break; + default: + EKAT_ERROR_MSG ("Layout not supported by CoarseningRemapper: " + e2str(lt) + "\n"); + } + return tgt; +} + +void HorizInterpRemapperBase::do_registration_ends () +{ + if (this->m_num_bound_fields==this->m_num_registered_fields) { + create_ov_fields (); + setup_mpi_data_structures (); + } +} + +void HorizInterpRemapperBase:: +do_register_field (const identifier_type& src, const identifier_type& tgt) +{ + constexpr auto COL = ShortFieldTagsNames::COL; + EKAT_REQUIRE_MSG (src.get_layout().has_tag(COL), + "Error! Cannot register a field without COL tag in RefiningRemapperP2P.\n" + " - field name: " + src.name() + "\n" + " - field layout: " + to_string(src.get_layout()) + "\n"); + m_src_fields.push_back(field_type(src)); + m_tgt_fields.push_back(field_type(tgt)); +} + +void HorizInterpRemapperBase:: +do_bind_field (const int ifield, const field_type& src, const field_type& tgt) +{ + EKAT_REQUIRE_MSG (src.data_type()==DataType::RealType, + "Error! RefiningRemapperRMA only allows fields with RealType data.\n" + " - src field name: " + src.name() + "\n" + " - src field type: " + e2str(src.data_type()) + "\n"); + EKAT_REQUIRE_MSG (tgt.data_type()==DataType::RealType, + "Error! RefiningRemapperRMA only allows fields with RealType data.\n" + " - tgt field name: " + tgt.name() + "\n" + " - tgt field type: " + e2str(tgt.data_type()) + "\n"); + + m_src_fields[ifield] = src; + m_tgt_fields[ifield] = tgt; + + // If this was the last field to be bound, we can setup the MPI schedule + if (this->m_state==RepoState::Closed && + (this->m_num_bound_fields+1)==this->m_num_registered_fields) { + create_ov_fields (); + setup_mpi_data_structures (); + } +} + +auto HorizInterpRemapperBase:: +get_my_triplets (const std::string& map_file) const + -> std::vector +{ + using gid_type = AbstractGrid::gid_type; + using namespace ShortFieldTagsNames; + + // 1. Load the map file chunking it evenly across all ranks + scorpio::register_file(map_file,scorpio::FileMode::Read); + + // 1.1 Create a "helper" grid, with as many dofs as the number + // of triplets in the map file, and divided linearly across ranks + const int ngweights = scorpio::get_dimlen(map_file,"n_s"); + int nlweights = ngweights / m_comm.size(); + if (m_comm.rank() < (ngweights % m_comm.size())) { + nlweights += 1; + } + + gid_type offset = nlweights; + m_comm.scan(&offset,1,MPI_SUM); + offset -= nlweights; // scan is inclusive, but we need exclusive + + // Create a unique decomp tag, which ensures all refining remappers have + // their own decomposition + static int tag_counter = 0; + const std::string int_decomp_tag = "RR::gmtg,int,grid-idx=" + std::to_string(tag_counter++); + const std::string real_decomp_tag = "RR::gmtg,real,grid-idx=" + std::to_string(tag_counter++); + + // 1.2 Read a chunk of triplets col indices + std::vector cols(nlweights); + std::vector rows(nlweights); + std::vector S(nlweights); + + scorpio::register_variable(map_file, "col", "col", {"n_s"}, "int", int_decomp_tag); + scorpio::register_variable(map_file, "row", "row", {"n_s"}, "int", int_decomp_tag); + scorpio::register_variable(map_file, "S", "S", {"n_s"}, "real", real_decomp_tag); + + std::vector dofs_offsets(nlweights); + std::iota(dofs_offsets.begin(),dofs_offsets.end(),offset); + scorpio::set_dof(map_file,"col",nlweights,dofs_offsets.data()); + scorpio::set_dof(map_file,"row",nlweights,dofs_offsets.data()); + scorpio::set_dof(map_file,"S" ,nlweights,dofs_offsets.data()); + scorpio::set_decomp(map_file); + + scorpio::grid_read_data_array(map_file,"col",-1,cols.data(),cols.size()); + scorpio::grid_read_data_array(map_file,"row",-1,rows.data(),rows.size()); + scorpio::grid_read_data_array(map_file,"S" ,-1,S.data(),S.size()); + + scorpio::eam_pio_closefile(map_file); + + // 1.3 Dofs in grid are likely 0-based, while row/col ids in map file + // are likely 1-based. To match dofs, we need to offset the row/cols + // ids we just read in. + int map_file_min_row = std::numeric_limits::max(); + int map_file_min_col = std::numeric_limits::max(); + for (int id=0; idget_global_min_dof_gid(); + } else { + col_offset -= m_fine_grid->get_global_min_dof_gid(); + } + for (auto& id : rows) { + id -= row_offset; + } + for (auto& id : cols) { + id -= col_offset; + } + + // Create a grid based on the row gids I read in (may be duplicated across ranks) + std::vector unique_gids; + const auto& gids = m_type==InterpType::Refine ? rows : cols; + for (auto gid : gids) { + if (not ekat::contains(unique_gids,gid)) { + unique_gids.push_back(gid); + } + } + auto io_grid = std::make_shared ("helper",unique_gids.size(),0,m_comm); + auto io_grid_gids_h = io_grid->get_dofs_gids().get_view(); + int k = 0; + for (auto gid : unique_gids) { + io_grid_gids_h(k++) = gid; + } + io_grid->get_dofs_gids().sync_to_dev(); + + // Create Triplets to export, sorted by gid + std::map> io_triplets; + auto io_grid_gid2lid = io_grid->get_gid2lid_map(); + for (int i=0; i(); + auto mpi_real_t = ekat::get_mpi_type(); + int lengths[3] = {1,1,1}; + MPI_Aint displacements[3] = {0, offsetof(Triplet,col), offsetof(Triplet,w)}; + MPI_Datatype types[3] = {mpi_gid_t,mpi_gid_t,mpi_real_t}; + MPI_Datatype mpi_triplet_t; + MPI_Type_create_struct (3,lengths,displacements,types,&mpi_triplet_t); + MPI_Type_commit(&mpi_triplet_t); + + // Create import-export + GridImportExport imp_exp (m_fine_grid,io_grid); + std::map> my_triplets_map; + imp_exp.gather(mpi_triplet_t,io_triplets,my_triplets_map); + MPI_Type_free(&mpi_triplet_t); + + std::vector my_triplets; + for (auto& it : my_triplets_map) { + my_triplets.reserve(my_triplets.size()+it.second.size()); + std::move(it.second.begin(),it.second.end(),std::back_inserter(my_triplets)); + } + + return my_triplets; +} + +void HorizInterpRemapperBase:: +create_coarse_grids (const std::vector& triplets) +{ + const int nlevs = m_fine_grid->get_num_vertical_levels(); + + // Gather overlapped coarse grid gids (rows or cols, depending on m_type) + std::map ov_gid2lid; + bool pickRow = m_type==InterpType::Coarsen; + for (const auto& t : triplets) { + ov_gid2lid.emplace(pickRow ? t.row : t.col,ov_gid2lid.size()); + } + int num_ov_gids = ov_gid2lid.size(); + + // Use a temp and then assing, b/c grid_ptr_type is a pointer to const, + // so you can't modify gids using that pointer + auto ov_coarse_grid = std::make_shared("ov_coarse_grid",num_ov_gids,nlevs,m_comm); + auto ov_coarse_gids_h = ov_coarse_grid->get_dofs_gids().get_view(); + for (const auto& it : ov_gid2lid) { + ov_coarse_gids_h[it.second] = it.first; + } + ov_coarse_grid->get_dofs_gids().sync_to_dev(); + m_ov_coarse_grid = ov_coarse_grid; + + // Create the unique coarse grid + auto coarse_gids = m_ov_coarse_grid->get_unique_gids(); + int num_gids = coarse_gids.size(); + m_coarse_grid = std::make_shared("coarse_grid",num_gids,nlevs,m_comm); + auto coarse_gids_h = m_coarse_grid->get_dofs_gids().get_view(); + std::copy(coarse_gids.begin(),coarse_gids.end(),coarse_gids_h.data()); + m_coarse_grid->get_dofs_gids().sync_to_dev(); +} + +void HorizInterpRemapperBase:: +create_crs_matrix_structures (std::vector& triplets) +{ + // Get row/col data depending on interp type + bool refine = m_type==InterpType::Refine; + auto row_grid = refine ? m_fine_grid : m_ov_coarse_grid; + auto col_grid = refine ? m_ov_coarse_grid : m_fine_grid; + const int num_rows = row_grid->get_num_local_dofs(); + + auto col_gid2lid = col_grid->get_gid2lid_map(); + auto row_gid2lid = row_grid->get_gid2lid_map(); + + // Sort triplets so that row GIDs appear in the same order as + // in the row grid. If two row GIDs are the same, use same logic + // with col + auto compare = [&] (const Triplet& lhs, const Triplet& rhs) { + auto lhs_lrow = row_gid2lid.at(lhs.row); + auto rhs_lrow = row_gid2lid.at(rhs.row); + auto lhs_lcol = col_gid2lid.at(lhs.col); + auto rhs_lcol = col_gid2lid.at(rhs.col); + return lhs_lrow("",num_rows+1); + m_col_lids = view_1d("",nnz); + m_weights = view_1d("",nnz); + + auto row_offsets_h = Kokkos::create_mirror_view(m_row_offsets); + auto col_lids_h = Kokkos::create_mirror_view(m_col_lids); + auto weights_h = Kokkos::create_mirror_view(m_weights); + + // Fill col ids and weights + for (int i=0; i row_counts(num_rows); + for (int i=0; iget_num_local_dofs(); + const auto ov_gn = m_ov_coarse_grid->name(); + const auto dt = DataType::RealType; + for (int i=0; i +void HorizInterpRemapperBase:: +local_mat_vec (const Field& x, const Field& y) const +{ + using RangePolicy = typename KT::RangePolicy; + using MemberType = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + using Pack = ekat::Pack; + using PackInfo = ekat::PackInfo; + + const auto row_grid = m_type==InterpType::Refine ? m_fine_grid : m_ov_coarse_grid; + const int nrows = row_grid->get_num_local_dofs(); + + const auto& src_layout = x.get_header().get_identifier().get_layout(); + const int rank = src_layout.rank(); + + auto row_offsets = m_row_offsets; + auto col_lids = m_col_lids; + auto weights = m_weights; + + switch (rank) { + // Note: in each case, handle 1st contribution to each row separately, + // using = instead of +=. This allows to avoid doing an extra + // loop to zero out y before the mat-vec. + case 1: + { + // Unlike get_view, get_strided_view returns a LayoutStride view, + // therefore allowing the 1d field to be a subfield of a 2d field + // along the 2nd dimension. + auto x_view = x.get_strided_view(); + auto y_view = y.get_strided_view< Real*>(); + Kokkos::parallel_for(RangePolicy(0,nrows), + KOKKOS_LAMBDA(const int& row) { + const auto beg = row_offsets(row); + const auto end = row_offsets(row+1); + y_view(row) = weights(beg)*x_view(col_lids(beg)); + for (int icol=beg+1; icol(); + auto y_view = y.get_view< Pack**>(); + const int dim1 = PackInfo::num_packs(src_layout.dim(1)); + auto policy = ESU::get_default_team_policy(nrows,dim1); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team) { + const auto row = team.league_rank(); + + const auto beg = row_offsets(row); + const auto end = row_offsets(row+1); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1), + [&](const int j){ + y_view(row,j) = weights(beg)*x_view(col_lids(beg),j); + for (int icol=beg+1; icol(); + auto y_view = y.get_view< Pack***>(); + const int dim1 = src_layout.dim(1); + const int dim2 = PackInfo::num_packs(src_layout.dim(2)); + auto policy = ESU::get_default_team_policy(nrows,dim1*dim2); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team) { + const auto row = team.league_rank(); + + const auto beg = row_offsets(row); + const auto end = row_offsets(row+1); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2), + [&](const int idx){ + const int j = idx / dim2; + const int k = idx % dim2; + y_view(row,j,k) = weights(beg)*x_view(col_lids(beg),j,k); + for (int icol=beg+1; icol(const Field&, const Field&) const; + +#if SCREAM_PACK_SIZE>1 +template +void HorizInterpRemapperBase:: +local_mat_vec(const Field&, const Field&) const; +#endif + +} // namespace scream diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp new file mode 100644 index 000000000000..9a91220d9a5d --- /dev/null +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp @@ -0,0 +1,139 @@ +#ifndef SCREAM_HORIZ_INTERP_REMAPPER_BASE_HPP +#define SCREAM_HORIZ_INTERP_REMAPPER_BASE_HPP + +#include "share/grid/remap/abstract_remapper.hpp" + +namespace scream +{ + +/* + * A base class for (horizontal) interpolation remappers + * + * This base class simply implements one method, common to all interpolation + * remappers, which reads a map file, and grabs the sparse matrix triplets + * that are needed. + */ + +class HorizInterpRemapperBase : public AbstractRemapper +{ +protected: + enum class InterpType { + Refine, + Coarsen + }; + +public: + HorizInterpRemapperBase (const grid_ptr_type& fine_grid, + const std::string& map_file, + const InterpType type); + + virtual ~HorizInterpRemapperBase () = default; + + FieldLayout create_src_layout (const FieldLayout& tgt_layout) const override; + FieldLayout create_tgt_layout (const FieldLayout& src_layout) const override; + + bool compatible_layouts (const layout_type& src, + const layout_type& tgt) const override { + // Same type of layout, and same sizes except for possibly the first one + // Note: we can't do [src|tgt].size()/[src|tgt].dim(0)), since there may + // be 0 src/tgt gids on some ranks, which means src/tgt.dim(0)=0. + using namespace ShortFieldTagsNames; + + return src.strip_dim(COL)==tgt.strip_dim(COL); + } + +protected: + + const identifier_type& do_get_src_field_id (const int ifield) const override { + return m_src_fields[ifield].get_header().get_identifier(); + } + const identifier_type& do_get_tgt_field_id (const int ifield) const override { + return m_tgt_fields[ifield].get_header().get_identifier(); + } + const field_type& do_get_src_field (const int ifield) const override { + return m_src_fields[ifield]; + } + const field_type& do_get_tgt_field (const int ifield) const override { + return m_tgt_fields[ifield]; + } + + void do_registration_begins () override { /* Nothing to do here */ } + void do_register_field (const identifier_type& src, const identifier_type& tgt) override; + void do_bind_field (const int ifield, const field_type& src, const field_type& tgt) override; + void do_registration_ends () override; + + void do_remap_bwd () override { + EKAT_ERROR_MSG ("HorizInterpRemapperBase only supports fwd remapping.\n"); + } + + using gid_type = AbstractGrid::gid_type; + using KT = KokkosTypes; + + template + using view_1d = typename KT::template view_1d; + + struct Triplet { + // Note: unfortunately, C++17 does not support emplace-ing POD + // types as aggregates unless a ctor is declared. C++20 does though. + Triplet () = default; + Triplet(const gid_type rr, const gid_type cc, const Real ww) + : row(rr), col(cc), w(ww) {} + gid_type row; + gid_type col; + Real w; + }; + + std::vector + get_my_triplets (const std::string& map_file) const; + + void create_coarse_grids (const std::vector& triplets); + + // Not a const ref, since we'll sort the triplets according to + // how row gids appear in the coarse grid + void create_crs_matrix_structures (std::vector& triplets); + + void create_ov_fields (); + + void clean_up (); + + // Derived classes will do different things, depending on m_type and the + // MPI strategy they use (P2P or RMA) + virtual void setup_mpi_data_structures () = 0; + +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + template + void local_mat_vec (const Field& f_src, const Field& f_tgt) const; + + // The fine and coarse grids. Depending on m_type, they could be + // respectively m_src_grid and m_tgt_grid or viceversa + // Note: coarse grid is non-const, so that we can add geo data later. + // This helps with m_type=Coarsen, which is typically during + // model output, so that we can coarsen also geo data. + grid_ptr_type m_fine_grid; + std::shared_ptr m_coarse_grid; + + // An version of the coarse grid where this rank owns all the ids + // needed for the local mat-vec product. Depending on m_type, this + // can be on the src or tgt side. + grid_ptr_type m_ov_coarse_grid; + + // Source, target, and overlapped intermediate fields + std::vector m_src_fields; + std::vector m_ov_fields; + std::vector m_tgt_fields; + + // ----- Sparse matrix CRS representation ---- // + view_1d m_row_offsets; + view_1d m_col_lids; + view_1d m_weights; + + InterpType m_type; + + ekat::Comm m_comm; +}; + +} // namespace scream + +#endif // SCREAM_HORIZ_INTERP_REMAPPER_BASE_HPP diff --git a/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp b/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp index a976bcf5a1d7..c9950f126cce 100644 --- a/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp +++ b/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp @@ -40,6 +40,12 @@ HorizontalMap::HorizontalMap(const ekat::Comm& comm, const std::string& map_name // S - will be used to populate a segment's "weights" void HorizontalMap::set_remap_segments_from_file(const std::string& remap_filename) { + // Ensure each horiz remap file gets a unique decomp tag + static std::map file2idx; + if (file2idx.find(remap_filename)==file2idx.end()) { + file2idx[remap_filename] = file2idx.size(); + } + start_timer("EAMxx::HorizontalMap::set_remap_segments_from_file"); // Open remap file and determine the amount of data to be read scorpio::register_file(remap_filename,scorpio::Read); @@ -75,8 +81,8 @@ void HorizontalMap::set_remap_segments_from_file(const std::string& remap_filena view_1d tgt_col("row",my_chunk); auto tgt_col_h = Kokkos::create_mirror_view(tgt_col); std::vector vec_of_dims = {"n_s"}; - std::string i_decomp = std::string("int-row-n_s-") + std::to_string(my_chunk); - scorpio::get_variable(remap_filename, "row", "row", vec_of_dims, "int", i_decomp); + std::string i_decomp = "HR::srsff,phase1,dt=int,n_s=" + std::to_string(my_chunk) + ",file-idx=" + std::to_string(file2idx[remap_filename]); + scorpio::register_variable(remap_filename, "row", "row", vec_of_dims, "int", i_decomp); std::vector var_dof(my_chunk); std::iota(var_dof.begin(),var_dof.end(),my_start); scorpio::set_dof(remap_filename,"row",var_dof.size(),var_dof.data()); @@ -145,11 +151,11 @@ void HorizontalMap::set_remap_segments_from_file(const std::string& remap_filena auto col_h = Kokkos::create_mirror_view(col); auto S_h = Kokkos::create_mirror_view(S); vec_of_dims = {"n_s"}; - i_decomp = std::string("int-col-n_s-") + std::to_string(var_dof.size()); - std::string r_decomp = std::string("Real-S-n_s-") + std::to_string(var_dof.size()); + i_decomp = "HR::srsff,phase2,dt=int,n_s=" + std::to_string(var_dof.size()) + ",file-idx=" + std::to_string(file2idx[remap_filename]); + std::string r_decomp = "HR::srsff,phase2,dt=real,n_s=" + std::to_string(var_dof.size()) + ",file-idx=" + std::to_string(file2idx[remap_filename]); scorpio::register_file(remap_filename,scorpio::Read); - scorpio::get_variable(remap_filename, "col", "col", vec_of_dims, "int", i_decomp); - scorpio::get_variable(remap_filename, "S", "S", vec_of_dims, "real", r_decomp); + scorpio::register_variable(remap_filename, "col", "col", vec_of_dims, "int", i_decomp); + scorpio::register_variable(remap_filename, "S", "S", vec_of_dims, "real", r_decomp); scorpio::set_dof(remap_filename,"col",var_dof.size(),var_dof.data()); scorpio::set_dof(remap_filename,"S",var_dof.size(),var_dof.data()); scorpio::set_decomp(remap_filename); diff --git a/components/eamxx/src/share/grid/remap/identity_remapper.hpp b/components/eamxx/src/share/grid/remap/identity_remapper.hpp index 8fe90af42ab8..6b6659c3f60e 100644 --- a/components/eamxx/src/share/grid/remap/identity_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/identity_remapper.hpp @@ -36,10 +36,18 @@ class IdentityRemapper : public AbstractRemapper ~IdentityRemapper () = default; FieldLayout create_src_layout (const FieldLayout& tgt_layout) const override { + EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), + "[IdentityRemapper] Error! Input target layout is not valid for this remapper.\n" + " - input layout: " + to_string(tgt_layout)); + // Src and tgt grids are the same, so return the input return tgt_layout; } FieldLayout create_tgt_layout (const FieldLayout& src_layout) const override { + EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), + "[IdentityRemapper] Error! Input source layout is not valid for this remapper.\n" + " - input layout: " + to_string(src_layout)); + // Src and tgt grids are the same, so return the input return src_layout; } diff --git a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp new file mode 100644 index 000000000000..b1c738fe9256 --- /dev/null +++ b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp @@ -0,0 +1,383 @@ +#include "refining_remapper_p2p.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/grid/grid_import_export.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/util/scream_utils.hpp" + +#include +#include + +#include + +namespace scream +{ + +RefiningRemapperP2P:: +RefiningRemapperP2P (const grid_ptr_type& tgt_grid, + const std::string& map_file) + : HorizInterpRemapperBase(tgt_grid,map_file,InterpType::Refine) +{ + // Nothing to do here +} + +RefiningRemapperP2P:: +~RefiningRemapperP2P () +{ + clean_up(); +} + +void RefiningRemapperP2P::do_remap_fwd () +{ + // Fire the recv requests right away, so that if some other ranks + // is done packing before us, we can start receiving their data + if (not m_recv_req.empty()) { + check_mpi_call(MPI_Startall(m_recv_req.size(),m_recv_req.data()), + "[RefiningRemapperP2P] starting persistent recv requests.\n"); + } + + // Do P2P communications + pack_and_send (); + recv_and_unpack (); + + // Perform local-mat vec + // Helpef function, to establish if a field can be handled with packs + auto can_pack_field = [](const Field& f) { + const auto& ap = f.get_header().get_alloc_properties(); + return (ap.get_last_extent() % SCREAM_PACK_SIZE) == 0; + }; + + // Loop over each field, perform mat-vec + constexpr auto COL = ShortFieldTagsNames::COL; + for (int i=0; i(f_ov,f_tgt); + } else { + local_mat_vec<1>(f_ov,f_tgt); + } + } + + // Wait for all sends to be completed + if (not m_send_req.empty()) { + check_mpi_call(MPI_Waitall(m_send_req.size(),m_send_req.data(), MPI_STATUSES_IGNORE), + "[RefiningRemapperP2P] waiting on persistent send requests.\n"); + } +} + +void RefiningRemapperP2P::setup_mpi_data_structures () +{ + using namespace ShortFieldTagsNames; + + const int nranks = m_comm.size(); + + // Get cumulative col size of each field (to be used to compute offsets) + m_fields_col_sizes_scan_sum.resize(m_num_fields+1,0); + for (int i=0; iget_num_local_dofs(); + + m_imp_exp = std::make_shared(m_src_grid,m_ov_coarse_grid); + + // We can now compute the offset of each pid in the recv buffer + m_pids_recv_offsets = view_1d("",nranks+1); + auto ncols_recv_h = m_imp_exp->num_imports_per_pid_h(); + auto pids_recv_offsets_h = Kokkos::create_mirror_view(m_pids_recv_offsets); + pids_recv_offsets_h[0] = 0; + for (int pid=0; pid("",nranks+1); + auto pids_send_offsets_h = Kokkos::create_mirror_view(m_pids_send_offsets); + auto ncols_send_h = m_imp_exp->num_exports_per_pid_h(); + pids_send_offsets_h[0] = 0; + for (int pid=0; pid(); + for (int pid=0; pid0) { + auto send_ptr = m_mpi_send_buffer.data() + pids_send_offsets_h(pid)*total_col_size; + auto send_count = ncols_send_h(pid)*total_col_size; + auto& req = m_send_req.emplace_back(); + MPI_Send_init (send_ptr, send_count, mpi_real, pid, + 0, mpi_comm, &req); + } + // Recv request + if (ncols_recv_h(pid)>0) { + auto recv_ptr = m_mpi_recv_buffer.data() + pids_recv_offsets_h(pid)*total_col_size; + auto recv_count = ncols_recv_h(pid)*total_col_size; + auto& req = m_recv_req.emplace_back(); + MPI_Recv_init (recv_ptr, recv_count, mpi_real, pid, + 0, mpi_comm, &req); + } + } +} + +void RefiningRemapperP2P::pack_and_send () +{ + using RangePolicy = typename KT::RangePolicy; + using TeamMember = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + + auto export_pids = m_imp_exp->export_pids(); + auto export_lids = m_imp_exp->export_lids(); + auto ncols_send = m_imp_exp->num_exports_per_pid(); + auto pids_send_offsets = m_pids_send_offsets; + auto send_buf = m_send_buffer; + const int num_exports = export_pids.size(); + const int total_col_size = m_fields_col_sizes_scan_sum.back(); + for (int ifield=0; ifield(); + auto pack = KOKKOS_LAMBDA(const int iexp) { + auto pid = export_pids(iexp); + auto icol = export_lids(iexp); + auto pid_offset = pids_send_offsets(pid); + auto pos_within_pid = iexp - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_send(pid)*f_col_sizes_scan_sum + + pos_within_pid; + send_buf(offset) = v(icol); + }; + Kokkos::parallel_for(RangePolicy(0,num_exports),pack); + break; + } + case 2: + { + const auto v = f.get_view(); + const int dim1 = fl.dim(1); + auto policy = ESU::get_default_team_policy(num_exports,dim1); + auto pack = KOKKOS_LAMBDA(const TeamMember& team) { + const int iexp = team.league_rank(); + const int icol = export_lids(iexp); + const int pid = export_pids(iexp); + auto pid_offset = pids_send_offsets(pid); + auto pos_within_pid = iexp - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_send(pid)*f_col_sizes_scan_sum + + pos_within_pid*dim1; + auto col_pack = [&](const int& k) { + send_buf(offset+k) = v(icol,k); + }; + auto tvr = Kokkos::TeamVectorRange(team,dim1); + Kokkos::parallel_for(tvr,col_pack); + }; + Kokkos::parallel_for(policy,pack); + break; + } + case 3: + { + const auto v = f.get_view(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int f_col_size = dim1*dim2; + auto policy = ESU::get_default_team_policy(num_exports,dim1*dim2); + auto pack = KOKKOS_LAMBDA(const TeamMember& team) { + const int iexp = team.league_rank(); + const int icol = export_lids(iexp); + const int pid = export_pids(iexp); + auto pid_offset = pids_send_offsets(pid); + auto pos_within_pid = iexp - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_send(pid)*f_col_sizes_scan_sum + + pos_within_pid*f_col_size; + auto col_pack = [&](const int& idx) { + const int j = idx / dim2; + const int k = idx % dim2; + send_buf(offset+idx) = v(icol,j,k); + }; + auto tvr = Kokkos::TeamVectorRange(team,f_col_size); + Kokkos::parallel_for(tvr,col_pack); + }; + Kokkos::parallel_for(policy,pack); + break; + } + default: + EKAT_ERROR_MSG ("Unexpected field rank in RefiningRemapperP2P::pack.\n" + " - MPI rank : " + std::to_string(m_comm.rank()) + "\n" + " - field name: " + f.name() + "\n" + " - field rank: " + std::to_string(fl.rank()) + "\n"); + } + } + + // Wait for all threads to be done packing + Kokkos::fence(); + + // If MPI does not use dev pointers, we need to deep copy from dev to host + if (not MpiOnDev) { + Kokkos::deep_copy (m_mpi_send_buffer,m_send_buffer); + } + + if (not m_send_req.empty()) { + check_mpi_call(MPI_Startall(m_send_req.size(),m_send_req.data()), + "[RefiningRemapperP2P] start persistent send requests.\n"); + } +} + +void RefiningRemapperP2P::recv_and_unpack () +{ + if (not m_recv_req.empty()) { + check_mpi_call(MPI_Waitall(m_recv_req.size(),m_recv_req.data(), MPI_STATUSES_IGNORE), + "[RefiningRemapperP2P] waiting on persistent recv requests.\n"); + } + // If MPI does not use dev pointers, we need to deep copy from host to dev + if (not MpiOnDev) { + Kokkos::deep_copy (m_recv_buffer,m_mpi_recv_buffer); + } + + using RangePolicy = typename KT::RangePolicy; + using TeamMember = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + + auto import_pids = m_imp_exp->import_pids(); + auto import_lids = m_imp_exp->import_lids(); + auto ncols_recv = m_imp_exp->num_imports_per_pid(); + auto pids_recv_offsets = m_pids_recv_offsets; + auto recv_buf = m_recv_buffer; + const int num_imports = import_pids.size(); + const int total_col_size = m_fields_col_sizes_scan_sum.back(); + for (int ifield=0; ifield(); + auto unpack = KOKKOS_LAMBDA (const int idx) { + const int pid = import_pids(idx); + const int icol = import_lids(idx); + const auto pid_offset = pids_recv_offsets(pid); + const auto pos_within_pid = idx - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_recv(pid)*f_col_sizes_scan_sum + + pos_within_pid; + v(icol) = recv_buf(offset); + }; + Kokkos::parallel_for(RangePolicy(0,num_imports),unpack); + break; + } + case 2: + { + auto v = f.get_view(); + const int dim1 = fl.dim(1); + auto policy = ESU::get_default_team_policy(num_imports,dim1); + auto unpack = KOKKOS_LAMBDA (const TeamMember& team) { + const int idx = team.league_rank(); + const int pid = import_pids(idx); + const int icol = import_lids(idx); + const auto pid_offset = pids_recv_offsets(pid); + const auto pos_within_pid = idx - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_recv(pid)*f_col_sizes_scan_sum + + pos_within_pid*dim1; + auto col_unpack = [&](const int& k) { + v(icol,k) = recv_buf(offset+k); + }; + auto tvr = Kokkos::TeamVectorRange(team,dim1); + Kokkos::parallel_for(tvr,col_unpack); + }; + Kokkos::parallel_for(policy,unpack); + break; + } + case 3: + { + auto v = f.get_view(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int f_col_size = dim1*dim2; + auto policy = ESU::get_default_team_policy(num_imports,dim1*dim2); + auto unpack = KOKKOS_LAMBDA (const TeamMember& team) { + const int idx = team.league_rank(); + const int pid = import_pids(idx); + const int icol = import_lids(idx); + const auto pid_offset = pids_recv_offsets(pid); + const auto pos_within_pid = idx - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_recv(pid)*f_col_sizes_scan_sum + + pos_within_pid*f_col_size; + auto col_unpack = [&](const int& idx) { + const int j = idx / dim2; + const int k = idx % dim2; + v(icol,j,k) = recv_buf(offset+idx); + }; + auto tvr = Kokkos::TeamVectorRange(team,f_col_size); + Kokkos::parallel_for(tvr,col_unpack); + }; + Kokkos::parallel_for(policy,unpack); + break; + } + default: + EKAT_ERROR_MSG ("Unexpected field rank in RefiningRemapperP2P::unpack.\n" + " - MPI rank : " + std::to_string(m_comm.rank()) + "\n" + " - field name: " + f.name() + "\n" + " - field rank: " + std::to_string(fl.rank()) + "\n"); + } + } +} + +void RefiningRemapperP2P::clean_up () +{ + // Clear all MPI related structures + m_send_buffer = view_1d(); + m_recv_buffer = view_1d(); + m_mpi_send_buffer = mpi_view_1d(); + m_mpi_recv_buffer = mpi_view_1d(); + m_send_req.clear(); + m_recv_req.clear(); + m_imp_exp = nullptr; + + HorizInterpRemapperBase::clean_up(); +} + +} // namespace scream diff --git a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.hpp b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.hpp new file mode 100644 index 000000000000..4d4caef10fcc --- /dev/null +++ b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.hpp @@ -0,0 +1,121 @@ +#ifndef SCREAM_REFINING_REMAPPER_P2P_HPP +#define SCREAM_REFINING_REMAPPER_P2P_HPP + +#include "share/grid/remap/abstract_remapper.hpp" +#include "share/grid/remap/horiz_interp_remapper_base.hpp" +#include "scream_config.h" + +#include "ekat/ekat_pack.hpp" + +#include + +namespace scream +{ + +class GridImportExport; + +/* + * A remapper to interpolate fields on a coarser grid + * + * This remapper loads an interpolation sparse matrix from a map file, + * and performs an interpolation form a fine to a coarse grid by means + * of a mat-vec product. The sparse matrix encodes the interpolation + * weights. So far, the map file is *assumed* to store the matrix in + * triplet format, with row/col indices starting from 1. + * + * The remapper takes a src grid and the name of the map file. From here, + * it creates the tgt grid, and all the internal structures needed for + * an efficient mat-vec product at runtime. + * + * The mat-vec is performed in two stages: + * 1. Perform a local mat-vec multiplication (on device), producing intermediate + * output fields that have "duplicated" entries (that is, 2+ MPI + * ranks could all own a piece of the result for the same dof). + * 2. Perform a pack-send-recv-unpack sequence via MPI, to accumulate + * partial results on the rank that owns the dof in the tgt grid. + * + * The class has to create temporaries for the intermediate fields. + * An obvious future development would be to use some scratch memory + * for these fields, so to not increase memory pressure. + * + * The setup of the class uses a bunch of RMA mpi operations, since they + * are more convenient when ranks don't know where data is coming from + * or how much data is coming from each rank. The runtime operations, + * however, use the classic send/recv paradigm, where data is packed in + * a buffer, sent to the recv rank, and then unpacked and accumulated + * into the result. + */ + +class RefiningRemapperP2P : public HorizInterpRemapperBase +{ +public: + + RefiningRemapperP2P (const grid_ptr_type& tgt_grid, + const std::string& map_file); + + ~RefiningRemapperP2P (); + +protected: + + void do_remap_fwd () override; + +protected: + + void setup_mpi_data_structures () override; + + // This class uses itself to remap src grid geo data to the tgt grid. But in order + // to not pollute the remapper for later use, we must be able to clean it up after + // remapping all the geo data. + void clean_up (); + +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void pack_and_send (); + void recv_and_unpack (); + +protected: + + // If MpiOnDev=true, we pass device pointers to MPI. Otherwise, we use host mirrors. + static constexpr bool MpiOnDev = SCREAM_MPI_ON_DEVICE; + template + using mpi_view_1d = typename std::conditional< + MpiOnDev, + view_1d, + typename view_1d::HostMirror + >::type; + + // ----- Data structures for pack/unpack and MPI ----- // + + // Exclusive scan sum of the col size of each field + std::vector m_fields_col_sizes_scan_sum; + + // ImportData/export info + std::shared_ptr m_imp_exp; + + // The send/recv buffers for pack/unpack operations + view_1d m_send_buffer; + view_1d m_recv_buffer; + + // The send/recv buf to feed to MPI. + // If MpiOnDev=true, they simply alias the ones above + mpi_view_1d m_mpi_send_buffer; + mpi_view_1d m_mpi_recv_buffer; + + // Offset of each pid in send/recv buffers + view_1d m_pids_send_offsets; + view_1d m_pids_recv_offsets; + + // For each col, its position within the set of cols + // sent/recv to/from the corresponding remote + view_1d m_send_col_pos; + view_1d m_recv_col_pos; + + // Send/recv persistent requests + std::vector m_send_req; + std::vector m_recv_req; +}; + +} // namespace scream + +#endif // SCREAM_REFINING_REMAPPER_P2P_HPP diff --git a/components/eamxx/src/share/grid/remap/refining_remapper_rma.cpp b/components/eamxx/src/share/grid/remap/refining_remapper_rma.cpp new file mode 100644 index 000000000000..8253734232c6 --- /dev/null +++ b/components/eamxx/src/share/grid/remap/refining_remapper_rma.cpp @@ -0,0 +1,169 @@ +#include "refining_remapper_rma.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/io/scorpio_input.hpp" + +#include +#include + +#include + +namespace scream +{ + +RefiningRemapperRMA:: +RefiningRemapperRMA (const grid_ptr_type& tgt_grid, + const std::string& map_file) + : HorizInterpRemapperBase(tgt_grid,map_file,InterpType::Refine) +{ + // Nothing to do here +} + +RefiningRemapperRMA:: +~RefiningRemapperRMA () +{ + clean_up(); +} + +void RefiningRemapperRMA::do_remap_fwd () +{ + // Start RMA epoch on each field + for (int i=0; i(); + for (int i=0; i(); + for (int icol=0; icolget_num_local_dofs(); ++icol) { + const int pid = m_remote_pids[icol]; + const int lid = m_remote_lids[icol]; + check_mpi_call(MPI_Get(ov_data+icol*col_size,col_size,dt,pid, + lid*col_stride+col_offset,col_size,dt,win), + "MPI_Get for field: " + m_ov_fields[i].name()); + } + } + + // Close access RMA epoch on each field (exposure is still open) + for (int i=0; i(f_ov_src,f_tgt); + } else { + local_mat_vec<1>(f_ov_src,f_tgt); + } + } + + // Close exposure RMA epoch on each field + for (int i=0; iget_dofs_gids().get_view(); + m_src_grid->get_remote_pids_and_lids(ov_src_gids,m_remote_pids,m_remote_lids); + + // TODO: scope out possibility of using sub-groups for start/post calls + // (but I'm afraid you can't, b/c start/post may require same groups) + + // Create per-field structures + m_mpi_win.resize(m_num_fields); + m_col_size.resize(m_num_fields); + m_col_stride.resize(m_num_fields); + m_col_offset.resize(m_num_fields,0); + for (int i=0; iget_parent().lock()==nullptr, + "Error! We do not support remapping of subfields of other subfields.\n"); + const auto& sv_info = fap.get_subview_info(); + m_col_offset[i] = sv_info.slice_idx * col_stride; + m_col_stride[i] = sv_info.dim_extent * col_stride; + win_size *= sv_info.dim_extent; + } + + auto data = f.get_internal_view_data(); + check_mpi_call(MPI_Win_create(data,win_size,sizeof(Real), + MPI_INFO_NULL,mpi_comm,&m_mpi_win[i]), + "[RefiningRemapperRMA::setup_mpi_data_structures] MPI_Win_create"); +#ifndef EKAT_MPI_ERRORS_ARE_FATAL + check_mpi_call(MPI_Win_set_errhandler(m_mpi_win[i],MPI_ERRORS_RETURN), + "[RefiningRemapperRMA::setup_mpi_data_structure] setting MPI_ERRORS_RETURN handler on MPI_Win"); +#endif + } +} + +void RefiningRemapperRMA::clean_up () +{ + // Clear all MPI related structures + if (m_mpi_group!=MPI_GROUP_NULL) { + check_mpi_call(MPI_Group_free(&m_mpi_group),"MPI_Group_free"); + m_mpi_group = MPI_GROUP_NULL; + } + for (auto& win : m_mpi_win) { + check_mpi_call(MPI_Win_free(&win),"MPI_Win_free"); + } + m_mpi_win.clear(); + m_remote_pids.clear(); + m_remote_lids.clear(); + m_col_size.clear(); + + HorizInterpRemapperBase::clean_up(); +} + +} // namespace scream diff --git a/components/eamxx/src/share/grid/remap/refining_remapper_rma.hpp b/components/eamxx/src/share/grid/remap/refining_remapper_rma.hpp new file mode 100644 index 000000000000..3c310e324b17 --- /dev/null +++ b/components/eamxx/src/share/grid/remap/refining_remapper_rma.hpp @@ -0,0 +1,111 @@ +#ifndef SCREAM_REFINING_REMAPPER_RMA_HPP +#define SCREAM_REFINING_REMAPPER_RMA_HPP + +#include "share/grid/remap/abstract_remapper.hpp" +#include "share/grid/remap/horiz_interp_remapper_base.hpp" +#include "share/util/scream_utils.hpp" +#include "scream_config.h" + +#include "ekat/ekat_pack.hpp" + +#include + +namespace scream +{ + +/* + * A remapper to interpolate fields on a finer grid + * + * This remapper loads an interpolation sparse matrix from a map file, + * and performs an interpolation form a coarse to a fine grid by means + * of a mat-vec product. The sparse matrix encodes the interpolation + * weights. So far, the map file is *assumed* to store the matrix in + * triplet format, with row/col indices starting from 1. + * + * The remapper takes a tgt grid and the name of the map file. From here, + * it creates the src grid, and all the internal structures needed for + * an efficient mat-vec product at runtime. + * + * The mat-vec is performed in two stages: + * 1. Import remote entries of the source fields into an overlapped + * partition, so that each rank has all the entries it needs to + * performe a local mat-vec product. + * 2. Perform the local mat-vec product (on device), producing using + * as input fields the onese produced by step one. + * + * The class has to create temporaries for the intermediate fields. + * An obvious future development would be to use some scratch memory + * for these fields, so to not increase memory pressure. + * + * All the MPI operations performed by this class are implemented with + * one-sided (or RMA) MPI routines. One-sided MPI has been in the MPI + * standard since 2.0, but its support is still sub-optimal, due to + * limited effort in optimizing it by the vendors. Furthermore, as of + * Oct 2023, RMA operations are not supported by GPU-aware implementations. + */ + +class RefiningRemapperRMA : public HorizInterpRemapperBase +{ +public: + + RefiningRemapperRMA (const grid_ptr_type& tgt_grid, + const std::string& map_file); + + ~RefiningRemapperRMA (); + +protected: + + void do_remap_fwd () override; + +protected: + + void setup_mpi_data_structures () override; + + // This class uses itself to remap src grid geo data to the tgt grid. But in order + // to not pollute the remapper for later use, we must be able to clean it up after + // remapping all the geo data. + void clean_up (); + +protected: + + // Wrap a pointer in an MPI_Win + template + MPI_Win get_mpi_window (T* v, int n) const { + MPI_Win win; + check_mpi_call (MPI_Win_create(v,n*sizeof(T),sizeof(T), + MPI_INFO_NULL,m_comm.mpi_comm(),&win), + "MPI_Win_create"); + return win; + } + + MPI_Group m_mpi_group = MPI_GROUP_NULL; + + // Unfortunately there is no GPU-aware mpi for RMA operations. + //static constexpr bool MpiOnDev = SCREAM_MPI_ON_DEVICE; + static constexpr bool MpiOnDev = false; + + // ------- MPI data structures -------- // + + // For each GID in m_ov_src_grid, store the pid it belongs + // to in m_src_grid, and the local id on that pid. + std::vector m_remote_pids; + std::vector m_remote_lids; + + // Column info for each field. + // Notes: + // - for subfields, col_stride!=col_size, otherwise they match + // - col_offset!=0 only for subfield that are not the 0-th entry along subf dim. + // - in general, col_data = col_stride*icol+col_offset. + // - strides/offsets are *only* for m_src_fields (ov_src are contiguous, and tgt are only + // accessed via get_view). + std::vector m_col_size; + std::vector m_col_stride; + std::vector m_col_offset; + + // One MPI window object for each field + std::vector m_mpi_win; +}; + +} // namespace scream + +#endif // SCREAM_REFINING_REMAPPER_RMA_HPP diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 9b1536876609..beda1f5d7850 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -54,46 +54,31 @@ VerticalRemapper (const grid_ptr_type& src_grid, // as the source field, but will have a different number of // vertical levels. scorpio::register_file(map_file,scorpio::FileMode::Read); - m_num_remap_levs = scorpio::get_dimlen(map_file,"nlevs"); - scorpio::eam_pio_closefile(map_file); + m_num_remap_levs = scorpio::get_dimlen(map_file,"lev"); - auto tgt_grid_gids = src_grid->get_unique_gids(); - const int ngids = tgt_grid_gids.size(); - auto tgt_grid = std::make_shared("vertical_remap_tgt_grid",ngids,m_num_remap_levs,m_comm); - auto tgt_grid_gids_h = tgt_grid->get_dofs_gids().get_view(); - std::memcpy(tgt_grid_gids_h.data(),tgt_grid_gids.data(),ngids*sizeof(gid_t)); - tgt_grid->get_dofs_gids().sync_to_dev(); + auto tgt_grid = src_grid->clone("vertical_remap_tgt_grid",true); + tgt_grid->reset_num_vertical_lev(m_num_remap_levs); this->set_grids(src_grid,tgt_grid); - // Replicate the src grid geo data in the tgt grid. - const auto& src_geo_data_names = src_grid->get_geometry_data_names(); - for (const auto& name : src_geo_data_names) { - const auto& src_data = src_grid->get_geometry_data(name); - const auto& src_data_fid = src_data.get_header().get_identifier(); - const auto& layout = src_data_fid.get_layout(); - // We only add geo data that is horizontal in nature, vertical geo data won't be added because the vertical structure - // has a rigid definition already. - if (layout.tags().back()!=LEV && layout.tags().back()!=ILEV) { - // Simply copy it in the tgt grid, but we still need to assign the new grid name. - FieldIdentifier tgt_data_fid(src_data_fid.name(),src_data_fid.get_layout(),src_data_fid.get_units(),m_tgt_grid->name()); - auto tgt_data = tgt_grid->create_geometry_data(tgt_data_fid); - tgt_data.deep_copy(src_data); - } - } - - // Set the LEV and ILEV vertical profiles for interpolation from - register_vertical_source_field(lev_prof,"mid"); - register_vertical_source_field(ilev_prof,"int"); + register_vertical_source_field(lev_prof); + register_vertical_source_field(ilev_prof); // Gather the pressure level data for vertical remapping set_pressure_levels(map_file); + + scorpio::eam_pio_closefile(map_file); } FieldLayout VerticalRemapper:: create_src_layout (const FieldLayout& tgt_layout) const { using namespace ShortFieldTagsNames; + + EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), + "[VerticalRemapper] Error! Input target layout is not valid for this remapper.\n" + " - input layout: " + to_string(tgt_layout)); + const auto lt = get_layout_type(tgt_layout.tags()); auto src = FieldLayout::invalid(); const bool midpoints = tgt_layout.has_tag(LEV); @@ -120,6 +105,11 @@ FieldLayout VerticalRemapper:: create_tgt_layout (const FieldLayout& src_layout) const { using namespace ShortFieldTagsNames; + + EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), + "[VerticalRemapper] Error! Input source layout is not valid for this remapper.\n" + " - input layout: " + to_string(src_layout)); + const auto lt = get_layout_type(src_layout.tags()); auto tgt = FieldLayout::invalid(); const bool midpoints = true; //src_layout.has_tag(LEV); @@ -144,8 +134,13 @@ create_tgt_layout (const FieldLayout& src_layout) const } void VerticalRemapper:: -set_pressure_levels(const std::string& map_file) { - scorpio::register_file(map_file,scorpio::FileMode::Read); +set_pressure_levels(const std::string& map_file) +{ + // Ensure each map file gets a different decomp name + static std::map file2idx; + if (file2idx.find(map_file)==file2idx.end()) { + file2idx[map_file] = file2idx.size(); + } using namespace ShortFieldTagsNames; std::vector tags = {LEV}; @@ -156,47 +151,39 @@ set_pressure_levels(const std::string& map_file) { m_remap_pres.get_header().get_alloc_properties().request_allocation(mPack::n); m_remap_pres.allocate_view(); - auto remap_pres_scal = m_remap_pres.get_view(); + auto remap_pres_scal = m_remap_pres.get_view(); std::vector dofs_offsets(m_num_remap_levs); std::iota(dofs_offsets.begin(),dofs_offsets.end(),0); - const std::string idx_decomp_tag = "vertical_remapper::" + std::to_string(m_num_remap_levs); - scorpio::get_variable(map_file, "p_levs", "p_levs", {"nlevs"}, "real", idx_decomp_tag); + const std::string decomp_tag = "VR::spl,nlev=" + std::to_string(m_num_remap_levs) + ",file-idx=" + std::to_string(file2idx[map_file]); + scorpio::register_variable(map_file, "p_levs", "p_levs", {"lev"}, "real", decomp_tag); scorpio::set_dof(map_file,"p_levs",m_num_remap_levs,dofs_offsets.data()); scorpio::set_decomp(map_file); scorpio::grid_read_data_array(map_file,"p_levs",-1,remap_pres_scal.data(),remap_pres_scal.size()); - scorpio::eam_pio_closefile(map_file); + m_remap_pres.sync_to_dev(); } void VerticalRemapper:: -register_vertical_source_field(const Field& src, const std::string& mode) +register_vertical_source_field(const Field& src) { using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG(mode=="mid" || mode=="int","Error: VerticalRemapper::register_vertical_source_field," - "mode arg must be 'mid' or 'int'\n"); - - auto src_fid = src.get_header().get_identifier(); - if (mode=="mid") { - auto layout = src_fid.get_layout(); - auto name = src_fid.name(); - EKAT_REQUIRE_MSG(ekat::contains(std::vector{LEV},layout.tags().back()), - "Error::VerticalRemapper::register_vertical_source_field,\n" - "mode = 'mid' expects a layour ending with LEV tag.\n" - " - field name : " + name + "\n" + + EKAT_REQUIRE_MSG(src.is_allocated(), + "Error! Vertical level source field is not yet allocated.\n" + " - field name: " + src.name() + "\n"); + + const auto& layout = src.get_header().get_identifier().get_layout(); + const auto vert_tag = layout.tags().back(); + EKAT_REQUIRE_MSG (vert_tag==LEV or vert_tag==ILEV, + "Error! Input vertical level field does not have a vertical level tag at the end.\n" + " - field name: " + src.name() + "\n" " - field layout: " + to_string(layout) + "\n"); - EKAT_REQUIRE_MSG(src.is_allocated(), "Error! LEV source field is not yet allocated.\n"); + + if (vert_tag==LEV) { m_src_mid = src; m_mid_set = true; - } else { // mode=="int" - auto layout = src_fid.get_layout(); - auto name = src_fid.name(); - EKAT_REQUIRE_MSG(ekat::contains(std::vector{ILEV},layout.tags().back()), - "Error::VerticalRemapper::register_vertical_source_field,\n" - "mode = 'int' expects a layour ending with ILEV tag.\n" - " - field name : " + name + "\n" - " - field layout: " + to_string(layout) + "\n"); - EKAT_REQUIRE_MSG(src.is_allocated(), "Error! ILEV source field is not yet allocated.\n"); + } else { m_src_int = src; m_int_set = true; } @@ -281,10 +268,11 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) Field mask_tgt_fld (mask_tgt_fid); mask_tgt_fld.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); mask_tgt_fld.allocate_view(); - auto tgt_extra = tgt.get_header().get_extra_data(); - EKAT_REQUIRE_MSG(!tgt_extra.count("mask_data"),"ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); + EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_data"), + "ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); f_tgt.get_header().set_extra_data("mask_data",mask_tgt_fld); - EKAT_REQUIRE_MSG(!tgt_extra.count("mask_value"),"ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); + EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_value"), + "ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); f_tgt.get_header().set_extra_data("mask_value",m_mask_val); m_src_masks.push_back(mask_src_fld); m_tgt_masks.push_back(mask_tgt_fld); @@ -293,19 +281,19 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) // If a field does not have LEV or ILEV it may still have mask tracking assigned from somewhere else. // In those cases we want to copy that mask tracking to the target field. // Note, we still make a new field to ensure it is defined on the target grid. - const auto src_extra = src.get_header().get_extra_data(); - if (src_extra.count("mask_data")) { - auto f_src_mask = ekat::any_cast(src_extra.at("mask_data")); + if (src.get_header().has_extra_data("mask_data")) { + auto f_src_mask = src.get_header().get_extra_data("mask_data"); FieldIdentifier mask_tgt_fid (f_src_mask.name(), f_src_mask.get_header().get_identifier().get_layout(), nondim, m_tgt_grid->name() ); Field mask_tgt_fld (mask_tgt_fid); mask_tgt_fld.allocate_view(); mask_tgt_fld.deep_copy(f_src_mask); auto& f_tgt = m_tgt_fields[ifield]; - auto tgt_extra = tgt.get_header().get_extra_data(); - EKAT_REQUIRE_MSG(!tgt_extra.count("mask_data"),"ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); + EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_data"), + "ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); f_tgt.get_header().set_extra_data("mask_data",mask_tgt_fld); - EKAT_REQUIRE_MSG(!tgt_extra.count("mask_value"),"ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); + EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_value"), + "ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); f_tgt.get_header().set_extra_data("mask_value",m_mask_val); } } @@ -349,11 +337,9 @@ void VerticalRemapper::do_remap_fwd () // There is nothing to do, this field cannot be vertically interpolated, // so just copy it over. Note, if this field has its own mask data make // sure that is copied too. - auto f_tgt_extra = f_tgt.get_header().get_extra_data(); - if (f_tgt_extra.count("mask_data")) { - auto f_src_extra = f_src.get_header().get_extra_data(); - auto f_tgt_mask = ekat::any_cast(f_tgt_extra.at("mask_data")); - auto f_src_mask = ekat::any_cast(f_src_extra.at("mask_data")); + if (f_tgt.get_header().has_extra_data("mask_data")) { + auto f_tgt_mask = f_tgt.get_header().get_extra_data("mask_data"); + auto f_src_mask = f_src.get_header().get_extra_data("mask_data"); f_tgt_mask.deep_copy(f_src_mask); } f_tgt.deep_copy(f_src); diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp index 1bc1da57446e..880d305b0638 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp @@ -70,7 +70,7 @@ class VerticalRemapper : public AbstractRemapper protected: - void register_vertical_source_field(const Field& src, const std::string& mode); + void register_vertical_source_field(const Field& src); const identifier_type& do_get_src_field_id (const int ifield) const override { return m_src_fields[ifield].get_header().get_identifier(); @@ -110,7 +110,7 @@ class VerticalRemapper : public AbstractRemapper protected: using KT = KokkosTypes; - using gid_t = AbstractGrid::gid_type; + using gid_type = AbstractGrid::gid_type; template using RPack = ekat::Pack; diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 42afa8479ec2..29a62763cec4 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -1,8 +1,9 @@ #include "share/io/scorpio_input.hpp" -#include "ekat/ekat_parameter_list.hpp" #include "share/io/scream_scorpio_interface.hpp" +#include + #include #include @@ -36,8 +37,6 @@ AtmosphereInput (const std::string& filename, auto& names = params.get>("Field Names",{}); auto fm = std::make_shared(grid); - fm->registration_begins(); - fm->registration_ends(); for (auto& f : fields) { fm->add_field(f); names.push_back(f.name()); @@ -45,6 +44,17 @@ AtmosphereInput (const std::string& filename, init(params,fm); } +AtmosphereInput:: +~AtmosphereInput () +{ + // In practice, this should always be true, but since we have a do-nothing default ctor, + // it is possible to create an instance without ever using it. Since finalize would + // attempt to close the pio file, we need to call it only if init happened. + if (m_inited_with_views || m_inited_with_fields) { + finalize(); + } +} + void AtmosphereInput:: init (const ekat::ParameterList& params, const std::shared_ptr& field_mgr) @@ -86,8 +96,8 @@ init (const ekat::ParameterList& params, EKAT_REQUIRE_MSG (host_views_1d.size()==layouts.size(), "Error! Input host views and layouts maps has different sizes.\n" - " Input size: " + std::to_string(host_views_1d.size()) + "\n" - " Expected size: " + std::to_string(m_fields_names.size()) + "\n"); + " host_views_1d size: " + std::to_string(host_views_1d.size()) + "\n" + " layouts size: " + std::to_string(layouts.size()) + "\n"); m_layouts = layouts; m_host_views_1d = host_views_1d; @@ -97,12 +107,10 @@ init (const ekat::ParameterList& params, for (const auto& it : m_layouts) { m_fields_names.push_back(it.first); EKAT_REQUIRE_MSG (m_host_views_1d.count(it.first)==1, - "Error! Input layouts and views maps do not store the same keys.\n"); + "Error! Input layouts and views maps do not store the same keys.\n" + " layout = " + it.first); } - // Set the host views - set_views(host_views_1d,layouts); - // Init scorpio internal structures init_scorpio_structures (); @@ -118,6 +126,21 @@ set_field_manager (const std::shared_ptr& field_mgr) EKAT_REQUIRE_MSG (field_mgr, "Error! Invalid field manager pointer.\n"); EKAT_REQUIRE_MSG (field_mgr->get_grid(), "Error! Field manager stores an invalid grid pointer.\n"); + // If resetting a field manager we want to check that the layouts of all fields are the same. + if (m_field_mgr) { + for (auto felem = m_field_mgr->begin(); felem != m_field_mgr->end(); felem++) { + auto name = felem->first; + auto field_curr = m_field_mgr->get_field(name); + auto field_new = field_mgr->get_field(name); + // Check Layouts + auto lay_curr = field_curr.get_header().get_identifier().get_layout(); + auto lay_new = field_new.get_header().get_identifier().get_layout(); + EKAT_REQUIRE_MSG(lay_curr==lay_new,"ERROR!! AtmosphereInput::set_field_manager - setting new field manager which has different layout for field " << name <<"\n" + << " Old Layout: " << to_string(lay_curr) << "\n" + << " New Layout: " << to_string(lay_new) << "\n"); + } + } + m_field_mgr = field_mgr; // Store grid and fm @@ -168,11 +191,6 @@ set_grid (const std::shared_ptr& grid) " - num global dofs: " + std::to_string(grid->get_num_global_dofs()) + "\n"); } - EKAT_REQUIRE_MSG(grid->get_comm().size()<=grid->get_num_global_dofs(), - "Error! PIO interface requires the size of the IO MPI group to be\n" - " no greater than the global number of columns.\n" - " Consider decreasing the size of IO MPI group.\n"); - // The grid is good. Store it. m_io_grid = grid; } @@ -184,6 +202,10 @@ set_grid (const std::shared_ptr& grid) // running eam_update_timesnap. void AtmosphereInput::read_variables (const int time_index) { + auto func_start = std::chrono::steady_clock::now(); + if (m_atm_logger) { + m_atm_logger->info("[EAMxx::scorpio_input] Reading variables from file:\n\t " + m_filename + " ...\n"); + } EKAT_REQUIRE_MSG (m_inited_with_views || m_inited_with_fields, "Error! Scorpio structures not inited yet. Did you forget to call 'init(..)'?\n"); @@ -296,30 +318,12 @@ void AtmosphereInput::read_variables (const int time_index) f.sync_to_dev(); } } -} - -void AtmosphereInput:: -set_views (const std::map& host_views_1d, - const std::map& layouts) -{ - EKAT_REQUIRE_MSG (host_views_1d.size()==layouts.size(), - "Error! Input host views and layouts maps has different sizes.\n" - " Input size: " + std::to_string(host_views_1d.size()) + "\n" - " Expected size: " + std::to_string(m_fields_names.size()) + "\n"); - - m_layouts = layouts; - m_host_views_1d = host_views_1d; - - // Loop over one of the two maps, store key in m_fields_names, - // and check that the two maps have the same keys - for (const auto& it : m_layouts) { - m_fields_names.push_back(it.first); - EKAT_REQUIRE_MSG (m_host_views_1d.count(it.first)==1, - "Error! Input layouts and views maps do not store the same keys.\n"); + auto func_finish = std::chrono::steady_clock::now(); + if (m_atm_logger) { + auto duration = std::chrono::duration_cast(func_finish - func_start)/1000.0; + m_atm_logger->info("[EAMxx::scorpio_input] Reading variables from file:\n\t " + m_filename + " ... done! (Elapsed time = " + std::to_string(duration.count()) +" seconds)\n"); } - - m_inited_with_views = true; -} +} /* ---------------------------------------------------------- */ void AtmosphereInput::finalize() @@ -360,8 +364,15 @@ void AtmosphereInput::register_variables() const auto& fp_precision = "real"; for (auto const& name : m_fields_names) { // Determine the IO-decomp and construct a vector of dimension ids for this variable: - auto vec_of_dims = get_vec_of_dims(m_layouts.at(name)); - auto io_decomp_tag = get_io_decomp(m_layouts.at(name)); + const auto& layout = m_layouts.at(name); + auto vec_of_dims = get_vec_of_dims(layout); + auto io_decomp_tag = get_io_decomp(layout); + + for (size_t i=0; iget_partitioned_dim_tag()==layout.tags()[i]; + auto dimlen = partitioned ? m_io_grid->get_partitioned_dim_global_size() : layout.dims()[i]; + scorpio::register_dimension(m_filename, vec_of_dims[i], vec_of_dims[i], dimlen, partitioned); + } // TODO: Reverse order of dimensions to match flip between C++ -> F90 -> PIO, // may need to delete this line when switching to full C++/C implementation. @@ -372,8 +383,8 @@ void AtmosphereInput::register_variables() // Currently the field_manager only stores Real variables so it is not an issue, // but in the future if non-Real variables are added we will want to accomodate that. //TODO: Should be able to simply inquire from the netCDF the dimensions for each variable. - scorpio::get_variable(m_filename, name, name, - vec_of_dims, fp_precision, io_decomp_tag); + scorpio::register_variable(m_filename, name, name, + vec_of_dims, fp_precision, io_decomp_tag); } } @@ -402,21 +413,18 @@ AtmosphereInput::get_vec_of_dims(const FieldLayout& layout) std::string AtmosphereInput:: get_io_decomp(const FieldLayout& layout) { - // Given a vector of dimensions names, create a unique decomp string to register with I/O - // Note: We are hard-coding for only REAL input here. - // TODO: would be to allow for other dtypes - std::string io_decomp_tag = (std::string("Real-") + m_io_grid->name() + "-" + - std::to_string(m_io_grid->get_num_global_dofs())); - auto dims_names = get_vec_of_dims(layout); - for (size_t i=0; iget_unique_grid_id()) + ",layout="; - return io_decomp_tag; + std::vector range(layout.rank()); + std::iota(range.begin(),range.end(),0); + auto tag_and_dim = [&](int i) { + return m_io_grid->get_dim_name(layout.tag(i)) + + std::to_string(layout.dim(i)); + }; + + decomp_tag += ekat::join (range, tag_and_dim,"-"); + + return decomp_tag; } /* ---------------------------------------------------------- */ @@ -434,6 +442,11 @@ void AtmosphereInput::set_degrees_of_freedom() std::vector AtmosphereInput::get_var_dof_offsets(const FieldLayout& layout) { + // It may be that this MPI ranks owns no chunk of the field + if (layout.size()==0) { + return {}; + } + std::vector var_dof(layout.size()); // Gather the offsets of the dofs of this variable w.r.t. the *global* array. diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index b474ba0e1c5a..d5f900ae114e 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -7,6 +7,7 @@ #include "share/grid/grids_manager.hpp" #include "ekat/ekat_parameter_list.hpp" +#include "ekat/logging/ekat_logger.hpp" /* The AtmosphereInput class handles all input streams to SCREAM. * It is important to note that there does not exist an InputManager, @@ -75,7 +76,7 @@ class AtmosphereInput const std::shared_ptr& grid, const std::vector& fields); - virtual ~AtmosphereInput () = default; + ~AtmosphereInput (); // --- Methods --- // // Initialize the class for reading into FieldManager-owned fields. @@ -103,10 +104,20 @@ class AtmosphereInput // Cleans up the class void finalize(); + // Getters + std::string get_filename() { return m_filename; } // Simple getter to query the filename for this stream. + + // Expose the ability to set field manager for cases like time_interpolation where we swap fields + // between field managers to avoid deep_copy. + void set_field_manager (const std::shared_ptr& field_mgr); + + // Option to add a logger + void set_logger(const std::shared_ptr& atm_logger) { + m_atm_logger = atm_logger; + } protected: void set_grid (const std::shared_ptr& grid); - void set_field_manager (const std::shared_ptr& field_mgr); void set_views (const std::map& host_views_1d, const std::map& layouts); void init_scorpio_structures (); @@ -132,6 +143,9 @@ class AtmosphereInput bool m_inited_with_fields = false; bool m_inited_with_views = false; + + // The logger to be used throughout the ATM to log message + std::shared_ptr m_atm_logger; }; // Class AtmosphereInput } //namespace scream diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 71dbfaa57f91..08b4b83410d5 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -4,6 +4,7 @@ #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/vertical_remapper.hpp" #include "share/util/scream_timing.hpp" +#include "share/field/field_utils.hpp" #include "ekat/util/ekat_units.hpp" #include "ekat/util/ekat_string_utils.hpp" @@ -38,6 +39,37 @@ void combine (const Real& new_val, Real& curr_val, const OutputAvgType avg_type) EKAT_KERNEL_ERROR_MSG ("Unexpected value for m_avg_type. Please, contact developers.\n"); } } +// This one covers cases where a variable might be masked. +KOKKOS_INLINE_FUNCTION +void combine_and_fill (const Real& new_val, Real& curr_val, Real& avg_coeff, const OutputAvgType avg_type, const Real fill_value) +{ + const bool new_fill = (avg_coeff == 0.0); + const bool curr_fill = curr_val == fill_value; + if (curr_fill && new_fill) { + // Then the value is already set to be filled and the new value doesn't change things. + return; + } else if (curr_fill) { + // Then the current value is filled but the new value will replace that for all cases. + curr_val = new_val; + } else { + switch (avg_type) { + case OutputAvgType::Instant: + curr_val = new_val; + break; + case OutputAvgType::Max: + curr_val = new_fill ? curr_val : ekat::impl::max(curr_val,new_val); + break; + case OutputAvgType::Min: + curr_val = new_fill ? curr_val : ekat::impl::min(curr_val,new_val); + break; + case OutputAvgType::Average: + curr_val += (new_fill ? 0.0 : new_val); + break; + default: + EKAT_KERNEL_ERROR_MSG ("Unexpected value for m_avg_type. Please, contact developers.\n"); + } + } +} // This helper function is used to make sure that the list of fields in // m_fields_names is a list of unique strings, otherwise throw an error. @@ -60,8 +92,6 @@ AtmosphereOutput (const ekat::Comm& comm, // Create a FieldManager with the input fields auto fm = std::make_shared (grid); - fm->registration_begins(); - fm->registration_ends(); for (auto f : fields) { fm->add_field(f); } @@ -80,7 +110,6 @@ AtmosphereOutput (const ekat::Comm& comm, init (); } - AtmosphereOutput:: AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, const std::shared_ptr& field_mgr, @@ -90,8 +119,18 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, { using vos_t = std::vector; - if (params.isParameter("Fill Value")) { - m_fill_value = static_cast(params.get("Fill Value")); + if (params.isParameter("fill_value")) { + m_fill_value = static_cast(params.get("fill_value")); + // If the fill_value is specified there is a good chance the user expects the average count to track filling. + m_track_avg_cnt = true; + } + if (params.isParameter("track_fill")) { + // Note, we do this after checking for fill_value to give users that opportunity to turn off fill tracking, even + // if they specify a specific fill value. + m_track_avg_cnt = params.get("track_fill"); + } + if (params.isParameter("fill_threshold")) { + m_avg_coeff_threshold = params.get("fill_threshold"); } // Figure out what kind of averaging is requested @@ -144,7 +183,7 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, } sort_and_check(m_fields_names); - // Check if remapping and if so create the appropriate remapper + // Check if remapping and if so create the appropriate remapper // Note: We currently support three remappers // - vertical remapping from file // - horizontal remapping from file @@ -163,8 +202,22 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, // Register any diagnostics needed by this output stream set_diagnostics(); + // Helper lambda, to copy io string attributes. This will be used if any + // remapper is created, to ensure atts set by atm_procs are not lost + auto transfer_io_str_atts = [&] (const Field& src, Field& tgt) { + const std::string io_string_atts_key ="io: string attributes"; + using stratts_t = std::map; + const auto& src_atts = src.get_header().get_extra_data(io_string_atts_key); + auto& dst_atts = tgt.get_header().get_extra_data(io_string_atts_key); + for (const auto& [name,val] : src_atts) { + dst_atts[name] = val; + } + }; + // Setup remappers - if needed - if (use_vertical_remap_from_file) { + if (use_vertical_remap_from_file) { + // When vertically remapping there is a chance that filled values will be present, so be sure to track these + m_track_avg_cnt = true; // We build a remapper, to remap fields from the fm grid to the io grid auto vert_remap_file = params.get("vertical_remap_file"); auto f_lev = get_field("p_mid","sim"); @@ -181,9 +234,14 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, const auto src = get_field(fname,"sim"); const auto tgt_fid = m_vert_remapper->create_tgt_fid(src.get_header().get_identifier()); const auto packsize = src.get_header().get_alloc_properties().get_largest_pack_size(); - io_fm->register_field(FieldRequest(tgt_fid,packsize)); + io_fm->register_field(FieldRequest(tgt_fid,packsize)); } io_fm->registration_ends(); + for (const auto& fname : m_fields_names) { + const auto& src = get_field(fname,"sim"); + auto& tgt = io_fm->get_field(fname); + transfer_io_str_atts (src,tgt); + } // Register all output fields in the remapper. m_vert_remapper->registration_begins(); @@ -236,6 +294,11 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, io_fm->register_field(FieldRequest(tgt_fid,packsize)); } io_fm->registration_ends(); + for (const auto& fname : m_fields_names) { + const auto& src = get_field(fname,"before_horizontal_remap"); + auto& tgt = io_fm->get_field(fname); + transfer_io_str_atts (src,tgt); + } // Register all output fields in the remapper. m_horiz_remapper->registration_begins(); @@ -254,7 +317,7 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, // Reset the IO field manager set_field_manager(io_fm,"io"); - } + } // Setup I/O structures init (); @@ -266,7 +329,9 @@ void AtmosphereOutput::restart (const std::string& filename) // Create an input stream on the fly, and init averaging data ekat::ParameterList res_params("Input Parameters"); res_params.set("Filename",filename); - res_params.set("Field Names",m_fields_names); + std::vector input_field_names = m_fields_names; + input_field_names.insert(input_field_names.end(),m_avg_cnt_names.begin(),m_avg_cnt_names.end()); + res_params.set("Field Names",input_field_names); AtmosphereInput hist_restart (res_params,m_io_grid,m_host_views_1d,m_layouts); hist_restart.read_variables(); @@ -287,21 +352,26 @@ void AtmosphereOutput::init() // Now that the fields have been gathered register the local views which will be used to determine output data to be written. register_views(); +} - -} // init -/*-----*/ void AtmosphereOutput:: run (const std::string& filename, - const bool is_write_step, + const bool output_step, const bool checkpoint_step, const int nsteps_since_last_output, const bool allow_invalid_fields) { // If we do INSTANT output, but this is not an write step, // we can immediately return + const bool is_write_step = output_step or checkpoint_step; if (not is_write_step and m_avg_type==OutputAvgType::Instant) { return; } + Real duration_write = 0.0; // Record of time spent writing output + if (is_write_step) { + if (m_atm_logger) { + m_atm_logger->info("[EAMxx::scorpio_output] Writing variables to file:\n\t " + filename + " ...\n"); + } + } using namespace scream::scorpio; @@ -341,11 +411,48 @@ run (const std::string& filename, stop_timer("EAMxx::IO::horiz_remap"); } + // Update all of the averaging count views (if needed) + // The strategy is as follows: + // For the update to the averaged value for this timestep we need to track if + // a point in a specific layout is "filled" or not. So we create a set of local + // temporary views for each layout that are either 0 or 1 depending on if the + // value is filled or unfilled. + // We then use these values to update the overall average count views for that layout. + if (m_track_avg_cnt && m_add_time_dim) { + // Note, we assume that all fields that share a layout are also masked/filled in the same + // way. If, we need to handle a case where only a subset of output variables are expected to + // be masked/filled then the recommendation is to request those variables in a separate output + // stream. + // We cycle through all fields and mark points that are filled/masked in the local views. First + // initialize them to 1 representing unfilled. + for (const auto& name : m_avg_cnt_names) { + auto& dev_view = m_local_tmp_avg_cnt_views_1d.at(name); + Kokkos::deep_copy(dev_view,1.0); + } + // Now we cycle through all the fields + for (const auto& name : m_fields_names) { + auto field = get_field(name,"io"); + auto lookup = m_field_to_avg_cnt_map.at(name); + auto dev_view = m_local_tmp_avg_cnt_views_1d.at(lookup); + update_avg_cnt_view(field,dev_view); + } + // Finally, we update the overall avg_cnt_views + for (const auto& name : m_avg_cnt_names) { + auto track_view = m_dev_views_1d.at(name); + auto local_view = m_local_tmp_avg_cnt_views_1d.at(name); + const auto layout = m_layouts.at(name); + KT::RangePolicy policy(0,layout.size()); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { + track_view(i) += local_view(i); + }); + } + } + // Take care of updating and possibly writing fields. for (auto const& name : m_fields_names) { // Get all the info for this field. auto field = get_field(name,"io"); - const auto& layout = m_layouts.at(name); + const auto& layout = m_layouts.at(field.name()); const auto& dims = layout.dims(); const auto rank = layout.rank(); @@ -374,7 +481,26 @@ run (const std::string& filename, KT::RangePolicy policy(0,layout.size()); const auto extents = layout.extents(); + // Averaging count data + // If we are not tracking the average count then we don't need to build the + // views for the average count, so we leave them as essentially empty. + auto avg_cnt_dims = dims; + auto avg_cnt_data = data; + if (m_track_avg_cnt && m_add_time_dim) { + const auto lookup = m_field_to_avg_cnt_map.at(name); + avg_cnt_data = m_local_tmp_avg_cnt_views_1d.at(lookup).data(); + } else { + for (auto& dim : avg_cnt_dims) { + dim = 1; + } + avg_cnt_data = nullptr; + } + auto avg_type = m_avg_type; + auto track_avg_cnt = m_track_avg_cnt; + auto add_time_dim = m_add_time_dim; + auto fill_value = m_fill_value; + auto avg_coeff_threshold = m_avg_coeff_threshold; // If the dev_view_1d is aliasing the field device view (must be Instant output), // then there's no point in copying from the field's view to dev_view if (not is_aliasing_field_view) { @@ -385,8 +511,13 @@ run (const std::string& filename, // handling a few more scenarios auto new_view_1d = field.get_strided_view(); auto avg_view_1d = view_Nd_dev<1>(data,dims[0]); + auto avg_coeff_1d = view_Nd_dev<1>(avg_cnt_data,avg_cnt_dims[0]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { - combine(new_view_1d(i), avg_view_1d(i),avg_type); + if (track_avg_cnt && add_time_dim) { + combine_and_fill(new_view_1d(i), avg_view_1d(i),avg_coeff_1d(i),avg_type,fill_value); + } else { + combine(new_view_1d(i), avg_view_1d(i),avg_type); + } }); break; } @@ -394,10 +525,15 @@ run (const std::string& filename, { auto new_view_2d = field.get_view(); auto avg_view_2d = view_Nd_dev<2>(data,dims[0],dims[1]); + auto avg_coeff_2d = view_Nd_dev<2>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j; unflatten_idx(idx,extents,i,j); - combine(new_view_2d(i,j), avg_view_2d(i,j),avg_type); + if (track_avg_cnt && add_time_dim) { + combine_and_fill(new_view_2d(i,j), avg_view_2d(i,j),avg_coeff_2d(i,j),avg_type,fill_value); + } else { + combine(new_view_2d(i,j), avg_view_2d(i,j),avg_type); + } }); break; } @@ -405,10 +541,15 @@ run (const std::string& filename, { auto new_view_3d = field.get_view(); auto avg_view_3d = view_Nd_dev<3>(data,dims[0],dims[1],dims[2]); + auto avg_coeff_3d = view_Nd_dev<3>(avg_cnt_data,dims[0],avg_cnt_dims[1],avg_cnt_dims[2]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k; unflatten_idx(idx,extents,i,j,k); - combine(new_view_3d(i,j,k), avg_view_3d(i,j,k),avg_type); + if (track_avg_cnt && add_time_dim) { + combine_and_fill(new_view_3d(i,j,k), avg_view_3d(i,j,k),avg_coeff_3d(i,j,k),avg_type,fill_value); + } else { + combine(new_view_3d(i,j,k), avg_view_3d(i,j,k),avg_type); + } }); break; } @@ -416,10 +557,15 @@ run (const std::string& filename, { auto new_view_4d = field.get_view(); auto avg_view_4d = view_Nd_dev<4>(data,dims[0],dims[1],dims[2],dims[3]); + auto avg_coeff_4d = view_Nd_dev<4>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l; unflatten_idx(idx,extents,i,j,k,l); - combine(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_type); + if (track_avg_cnt && add_time_dim) { + combine_and_fill(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_coeff_4d(i,j,k,l),avg_type,fill_value); + } else { + combine(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_type); + } }); break; } @@ -427,10 +573,15 @@ run (const std::string& filename, { auto new_view_5d = field.get_view(); auto avg_view_5d = view_Nd_dev<5>(data,dims[0],dims[1],dims[2],dims[3],dims[4]); + auto avg_coeff_5d = view_Nd_dev<5>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3],avg_cnt_dims[4]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l,m; unflatten_idx(idx,extents,i,j,k,l,m); - combine(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_type); + if (track_avg_cnt && add_time_dim) { + combine_and_fill(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_coeff_5d(i,j,k,l,m),avg_type,fill_value); + } else { + combine(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_type); + } }); break; } @@ -438,10 +589,15 @@ run (const std::string& filename, { auto new_view_6d = field.get_view(); auto avg_view_6d = view_Nd_dev<6>(data,dims[0],dims[1],dims[2],dims[3],dims[4],dims[5]); + auto avg_coeff_6d = view_Nd_dev<6>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3],avg_cnt_dims[4],avg_cnt_dims[5]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l,m,n; unflatten_idx(idx,extents,i,j,k,l,m,n); - combine(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n),avg_type); + if (track_avg_cnt && add_time_dim) { + combine_and_fill(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n), avg_coeff_6d(i,j,k,l,m,n),avg_type,fill_value); + } else { + combine(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n),avg_type); + } }); break; } @@ -451,16 +607,54 @@ run (const std::string& filename, } if (is_write_step) { - if (avg_type==OutputAvgType::Average) { - // Divide by steps count only when the summation is complete - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { - data[i] /= nsteps_since_last_output; - }); + if (output_step and avg_type==OutputAvgType::Average) { + if (m_track_avg_cnt && m_add_time_dim) { + const auto avg_cnt_lookup = m_field_to_avg_cnt_map.at(name); + const auto avg_cnt_view = m_dev_views_1d.at(avg_cnt_lookup); + const auto avg_nsteps = avg_cnt_view.data(); + // Divide by steps count only when the summation is complete + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { + Real coeff_percentage = Real(avg_nsteps[i])/nsteps_since_last_output; + if (data[i] != fill_value && coeff_percentage > avg_coeff_threshold) { + data[i] /= avg_nsteps[i]; + } else { + data[i] = fill_value; + } + }); + } else { + // Divide by steps count only when the summation is complete + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { + data[i] /= nsteps_since_last_output; + }); + } } // Bring data to host auto view_host = m_host_views_1d.at(name); Kokkos::deep_copy (view_host,view_dev); + auto func_start = std::chrono::steady_clock::now(); + grid_write_data_array(filename,name,view_host.data(),view_host.size()); + auto func_finish = std::chrono::steady_clock::now(); + auto duration_loc = std::chrono::duration_cast(func_finish - func_start); + duration_write += duration_loc.count(); + } + } + // Handle writing the average count variables to file + if (is_write_step) { + for (const auto& name : m_avg_cnt_names) { + auto& view_dev = m_dev_views_1d.at(name); + // Bring data to host + auto view_host = m_host_views_1d.at(name); + Kokkos::deep_copy (view_host,view_dev); + auto func_start = std::chrono::steady_clock::now(); grid_write_data_array(filename,name,view_host.data(),view_host.size()); + auto func_finish = std::chrono::steady_clock::now(); + auto duration_loc = std::chrono::duration_cast(func_finish - func_start); + duration_write += duration_loc.count(); + } + } + if (is_write_step) { + if (m_atm_logger) { + m_atm_logger->info("[EAMxx::scorpio_output] Writing variables to file:\n\t " + filename + " ...done! (Elapsed time = " + std::to_string(duration_write/1000.0) +" seconds)\n"); } } } // run @@ -559,19 +753,13 @@ set_grid (const std::shared_ptr& grid) (grid->get_global_max_dof_gid()-grid->get_global_min_dof_gid()+1)==grid->get_num_global_dofs(), "Error! In order for IO to work, the grid must (globally) have dof gids in interval [gid_0,gid_0+num_global_dofs).\n"); -//ASD IS THIS STILL TRUE -//ASD EKAT_REQUIRE_MSG(m_comm.size()<=grid->get_num_global_dofs(), -//ASD "Error! PIO interface requires the size of the IO MPI group to be\n" -//ASD " no greater than the global number of columns.\n" -//ASD " Consider decreasing the size of IO MPI group.\n"); - // The grid is good. Store it. m_io_grid = grid; } void AtmosphereOutput::register_dimensions(const std::string& name) { -/* +/* * Checks that the dimensions associated with a specific variable will be registered with IO file. * INPUT: * field_manager: is a pointer to the field_manager for this simulation. @@ -582,7 +770,7 @@ void AtmosphereOutput::register_dimensions(const std::string& name) // Store the field layout const auto& fid = get_field(name,"io").get_header().get_identifier(); const auto& layout = fid.get_layout(); - m_layouts.emplace(name,layout); + m_layouts.emplace(fid.name(),layout); // Now check taht all the dims of this field are already set to be registered. for (int i=0; iget_partitioned_dim_tag()==tags[i]; if (tag_loc == m_dims.end()) { int tag_len = 0; - if(tags[i] == m_io_grid->get_partitioned_dim_tag()) { + if(is_partitioned) { // This is the dimension that is partitioned across ranks. tag_len = m_io_grid->get_partitioned_dim_global_size(); } else { tag_len = layout.dim(i); } m_dims[tag_name] = std::make_pair(tag_len,is_partitioned); - } else { + } else { EKAT_REQUIRE_MSG(m_dims.at(tag_name).first==dims[i] or is_partitioned, - "Error! Dimension " + tag_name + " on field " + name + " has conflicting lengths"); + "Error! Dimension " + tag_name + " on field " + name + " has conflicting lengths. " + "If same name applies to different dims (e.g. PhysicsGLL and PhysicsPG2 define " + "\"ncol\" at different lengths), reset tag name for one of the grids.\n"); } } } // register_dimensions @@ -635,7 +825,8 @@ void AtmosphereOutput::register_views() field.get_header().get_parent().expired() && not is_diagnostic; - const auto size = m_layouts.at(name).size(); + const auto layout = m_layouts.at(field.name()); + const auto size = layout.size(); if (can_alias_field_view) { // Alias field's data, to save storage. m_dev_views_1d.emplace(name,view_1d_dev(field.get_internal_view_data(),size)); @@ -644,18 +835,62 @@ void AtmosphereOutput::register_views() // Create a local view. m_dev_views_1d.emplace(name,view_1d_dev("",size)); m_host_views_1d.emplace(name,Kokkos::create_mirror(m_dev_views_1d[name])); - } + + // Now create and store a dev view to track the averaging count for this layout (if we are tracking) + // We don't need to track average counts for files that are not tracking the time dim + set_avg_cnt_tracking(name,"",layout); } + // Initialize the local views reset_dev_views(); } /* ---------------------------------------------------------- */ +void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const std::string& avg_cnt_suffix, const FieldLayout& layout) +{ + // Make sure this field "name" hasn't already been regsitered with avg_cnt tracking. + // Note, we check this because some diagnostics need to have their own tracking which + // is created at the 'create_diagnostics' function. + if (m_field_to_avg_cnt_map.count(name)>0) { + return; + } + + // If the field is not an output field, do not register the avg count. This can happen + // if a diag depends on another diag. In this case, the inner diag is never outputed, + // so we don't want to create an avg count for its layout, since it may contain dims + // that are not in the list of registered dims (the dims of the output vars). + // See issue https://github.com/E3SM-Project/scream/issues/2663 + if (not ekat::contains(m_fields_names,name)) { + return; + } + + // Now create and store a dev view to track the averaging count for this layout (if we are tracking) + // We don't need to track average counts for files that are not tracking the time dim + const auto size = layout.size(); + const auto tags = layout.tags(); + if (m_add_time_dim && m_track_avg_cnt) { + std::string avg_cnt_name = "avg_count" + avg_cnt_suffix; + for (int ii=0; iiget_dim_name(layout.tag(ii)); + avg_cnt_name += "_" + tag_name; + } + if (std::find(m_avg_cnt_names.begin(),m_avg_cnt_names.end(),avg_cnt_name)==m_avg_cnt_names.end()) { + m_avg_cnt_names.push_back(avg_cnt_name); + } + m_field_to_avg_cnt_map.emplace(name,avg_cnt_name); + m_dev_views_1d.emplace(avg_cnt_name,view_1d_dev("",size)); // Note, emplace will only add a new key if one isn't already there + m_local_tmp_avg_cnt_views_1d.emplace(avg_cnt_name,view_1d_dev("",size)); // Note, emplace will only add a new key if one isn't already there + m_host_views_1d.emplace(avg_cnt_name,Kokkos::create_mirror(m_dev_views_1d[avg_cnt_name])); + m_layouts.emplace(avg_cnt_name,layout); + } +} +/* ---------------------------------------------------------- */ void AtmosphereOutput:: reset_dev_views() { // Reset the local device views depending on the averaging type // Init dev view with an "identity" for avg_type + const Real fill_for_average = (m_track_avg_cnt && m_add_time_dim) ? m_fill_value : 0.0; for (auto const& name : m_fields_names) { switch (m_avg_type) { case OutputAvgType::Instant: @@ -668,59 +903,83 @@ reset_dev_views() Kokkos::deep_copy(m_dev_views_1d[name],std::numeric_limits::infinity()); break; case OutputAvgType::Average: - Kokkos::deep_copy(m_dev_views_1d[name],0); + Kokkos::deep_copy(m_dev_views_1d[name],fill_for_average); break; default: EKAT_ERROR_MSG ("Unrecognized averaging type.\n"); } } + // Reset all views for averaging count to 0 + for (auto const& name : m_avg_cnt_names) { + Kokkos::deep_copy(m_dev_views_1d[name],0); + } } /* ---------------------------------------------------------- */ void AtmosphereOutput:: register_variables(const std::string& filename, - const std::string& fp_precision) + const std::string& fp_precision, + const scorpio::FileMode mode) { using namespace scorpio; using namespace ShortFieldTagsNames; + using strvec_t = std::vector; - // Cycle through all fields and register. - for (auto const& name : m_fields_names) { - auto field = get_field(name,"io"); - auto& fid = field.get_header().get_identifier(); - // Make a unique tag for each decomposition. To reuse decomps successfully, - // we must be careful to make the tags 1-1 with the intended decomp. Here we - // use the I/O grid name and its global #DOFs, then append the local - // dimension data. - // We use real here because the data type for the decomp is the one used - // in the simulation and not the one used in the output file. - std::string io_decomp_tag = (std::string("Real-") + m_io_grid->name() + "-" + - std::to_string(m_io_grid->get_num_global_dofs())); + EKAT_REQUIRE_MSG (ekat::contains(strvec_t{"float","single","double","real"},fp_precision), + "Error! Invalid/unsupported value for fp_precision.\n" + " - input value: " + fp_precision + "\n" + " - supported values: float, single, double, real\n"); + + // Helper lambdas + auto set_decomp_tag = [&](const FieldLayout& layout) { + std::string decomp_tag = "dt=real,grid-idx=" + std::to_string(m_io_grid->get_unique_grid_id()) + ",layout="; + + std::vector range(layout.rank()); + std::iota(range.begin(),range.end(),0); + auto tag_and_dim = [&](int i) { + return m_io_grid->get_dim_name(layout.tag(i)) + + std::to_string(layout.dim(i)); + }; + + decomp_tag += ekat::join (range, tag_and_dim,"-"); + + if (m_add_time_dim) { + decomp_tag += "-time"; + } + return decomp_tag; + }; + + auto set_vec_of_dims = [&](const FieldLayout& layout) { std::vector vec_of_dims; - const auto& layout = fid.get_layout(); - std::string units = to_string(fid.get_units()); - for (int i=0; iget_dim_name(layout.tag(i)); if (layout.tag(i)==CMP) { tag_name += std::to_string(layout.dim(i)); } - // Concatenate the dimension string to the io-decomp string - io_decomp_tag += "-" + tag_name; - // If tag==CMP, we already attached the length to the tag name - if (layout.tag(i)!=ShortFieldTagsNames::CMP) { - io_decomp_tag += "_" + std::to_string(layout.dim(i)); - } vec_of_dims.push_back(tag_name); // Add dimensions string to vector of dims. } - // TODO: Reverse order of dimensions to match flip between C++ -> F90 -> PIO, // may need to delete this line when switching to fully C++/C implementation. std::reverse(vec_of_dims.begin(),vec_of_dims.end()); if (m_add_time_dim) { - io_decomp_tag += "-time"; vec_of_dims.push_back("time"); //TODO: See the above comment on time. - } else { - io_decomp_tag += "-notime"; } + return vec_of_dims; + }; + + // Cycle through all fields and register. + for (auto const& name : m_fields_names) { + auto field = get_field(name,"io"); + auto& fid = field.get_header().get_identifier(); + // Make a unique tag for each decomposition. To reuse decomps successfully, + // we must be careful to make the tags 1-1 with the intended decomp. Here we + // use the I/O grid name and its global #DOFs, then append the local + // dimension data. + // We use real here because the data type for the decomp is the one used + // in the simulation and not the one used in the output file. + const auto& layout = fid.get_layout(); + const auto& io_decomp_tag = set_decomp_tag(layout); + auto vec_of_dims = set_vec_of_dims(layout); + std::string units = fid.get_units().get_string(); // TODO Need to change dtype to allow for other variables. // Currently the field_manager only stores Real variables so it is not an issue, @@ -729,24 +988,58 @@ register_variables(const std::string& filename, register_variable(filename, name, name, units, vec_of_dims, "real",fp_precision, io_decomp_tag); - // Add any extra attributes for this variable, examples include: - // 1. A list of subfields associated with a field group output - // 2. A CF longname (TODO) - // First check if this is a field group w/ subfields. - const auto& children = field.get_header().get_children(); - if (children.size()>0) { - // This field is a parent to a set of subfields - std::string children_list; - children_list += "[ "; - for (const auto& ch_w : children) { - auto child = ch_w.lock(); - children_list += child->get_identifier().name() + ", "; + // Add any extra attributes for this variable + if (mode != FileMode::Append ) { + // Add FillValue as an attribute of each variable + // FillValue is a protected metadata, do not add it if it already existed + if (fp_precision=="double" or + (fp_precision=="real" and std::is_same::value)) { + double fill_value = m_fill_value; + set_variable_metadata(filename, name, "_FillValue",fill_value); + } else { + float fill_value = m_fill_value; + set_variable_metadata(filename, name, "_FillValue",fill_value); + } + + // If this is has subfields, add list of its children + const auto& children = field.get_header().get_children(); + if (children.size()>0) { + // This field is a parent to a set of subfields + std::string children_list; + children_list += "[ "; + for (const auto& ch_w : children) { + auto child = ch_w.lock(); + children_list += child->get_identifier().name() + ", "; + } + // Replace last "," with "]" + children_list.pop_back(); + children_list.pop_back(); + children_list += " ]"; + set_variable_metadata(filename,name,"sub_fields",children_list); } - // Replace last "," with "]" - children_list.pop_back(); - children_list.pop_back(); - children_list += " ]"; - set_variable_metadata(filename,name,"sub_fields",children_list); + + // If tracking average count variables then add the name of the tracking variable for this variable + if (m_track_avg_cnt && m_add_time_dim) { + const auto lookup = m_field_to_avg_cnt_map.at(name); + set_variable_metadata(filename,name,"averaging_count_tracker",lookup); + } + + // Atm procs may have set some request for metadata. + using stratts_t = std::map; + const auto& str_atts = field.get_header().get_extra_data("io: string attributes"); + for (const auto& [att_name,att_val] : str_atts) { + set_variable_metadata(filename,name,att_name,att_val); + } + } + } + // Now register the average count variables + if (m_track_avg_cnt && m_add_time_dim) { + for (const auto& name : m_avg_cnt_names) { + const auto layout = m_layouts.at(name); + auto io_decomp_tag = set_decomp_tag(layout); + auto vec_of_dims = set_vec_of_dims(layout); + register_variable(filename, name, name, "unitless", vec_of_dims, + "real",fp_precision, io_decomp_tag); } } } // register_variables @@ -754,6 +1047,11 @@ register_variables(const std::string& filename, std::vector AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) { + // It may be that this MPI rank owns no chunk of the field + if (layout.size()==0) { + return {}; + } + std::vector var_dof(layout.size()); // Gather the offsets of the dofs of this variable w.r.t. the *global* array. @@ -775,9 +1073,6 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) auto dofs_h = m_io_grid->get_dofs_gids().get_view(); if (layout.has_tag(ShortFieldTagsNames::COL)) { const int num_cols = m_io_grid->get_num_local_dofs(); - if (num_cols==0) { - return var_dof; - } // Note: col_size might be *larger* than the number of vertical levels, or even smaller. // E.g., (ncols,2,nlevs), or (ncols,2) respectively. @@ -802,9 +1097,6 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) const int num_my_elems = layout2d.dim(0); const int ngp = layout2d.dim(1); const int num_cols = num_my_elems*ngp*ngp; - if (num_cols==0) { - return var_dof; - } // Note: col_size might be *larger* than the number of vertical levels, or even smaller. // E.g., (ncols,2,nlevs), or (ncols,2) respectively. @@ -829,9 +1121,9 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) } else { // This field is *not* defined over columns, so it is not partitioned. std::iota(var_dof.begin(),var_dof.end(),0); - } + } - return var_dof; + return var_dof; } /* ---------------------------------------------------------- */ void AtmosphereOutput::set_degrees_of_freedom(const std::string& filename) @@ -847,15 +1139,23 @@ void AtmosphereOutput::set_degrees_of_freedom(const std::string& filename) set_dof(filename,name,var_dof.size(),var_dof.data()); m_dofs.emplace(std::make_pair(name,var_dof.size())); } + // Cycle through the average count fields and set degrees of freedom + for (auto const& name : m_avg_cnt_names) { + const auto layout = m_layouts.at(name); + auto var_dof = get_var_dof_offsets(layout); + set_dof(filename,name,var_dof.size(),var_dof.data()); + m_dofs.emplace(std::make_pair(name,var_dof.size())); + } - /* TODO: + /* TODO: * Gather DOF info directly from grid manager */ } // set_degrees_of_freedom /* ---------------------------------------------------------- */ void AtmosphereOutput:: setup_output_file(const std::string& filename, - const std::string& fp_precision) + const std::string& fp_precision, + const scorpio::FileMode mode) { using namespace scream::scorpio; @@ -865,7 +1165,7 @@ setup_output_file(const std::string& filename, } // Register variables with netCDF file. Must come after dimensions are registered. - register_variables(filename,fp_precision); + register_variables(filename,fp_precision,mode); // Set the offsets of the local dofs in the global vector. set_degrees_of_freedom(filename); @@ -903,6 +1203,15 @@ compute_diagnostic(const std::string& name, const bool allow_invalid_fields) // Either allow_invalid_fields=false, or all inputs are valid. Proceed. diag->compute_diagnostic(); + + // The diag may have failed to compute (e.g., t=0 output with a flux-like diag). + // If we're allowing invalid fields, then we should simply set diag=m_fill_value + if (allow_invalid_fields) { + auto d = diag->get_diagnostic(); + if (not d.get_header().get_tracking().get_time_stamp().is_valid()) { + d.deep_copy(m_fill_value); + } + } } /* ---------------------------------------------------------- */ // General get_field routine for output. @@ -910,17 +1219,17 @@ compute_diagnostic(const std::string& name, const bool allow_invalid_fields) // manager. If not it will next check to see if it is in the list // of available diagnostics. If neither of these two options it // will throw an error. -Field AtmosphereOutput::get_field(const std::string& name, const std::string mode) const +Field AtmosphereOutput:: +get_field(const std::string& name, const std::string& mode) const { const auto field_mgr = get_field_manager(mode); const auto sim_field_mgr = get_field_manager("sim"); + const bool can_be_diag = field_mgr == sim_field_mgr; if (field_mgr->has_field(name)) { return field_mgr->get_field(name); - } else if (m_diagnostics.find(name) != m_diagnostics.end() && field_mgr==sim_field_mgr) { + } else if (m_diagnostics.find(name) != m_diagnostics.end() && can_be_diag) { const auto& diag = m_diagnostics.at(name); return diag->get_diagnostic(); - } else if (m_fields_alt_name.find(name) != m_fields_alt_name.end()) { - return get_field(m_fields_alt_name.at(name),mode); } else { EKAT_ERROR_MSG ("ERROR::AtmosphereOutput::get_field Field " + name + " not found in " + mode + " field manager or diagnostics list."); } @@ -930,111 +1239,234 @@ void AtmosphereOutput::set_diagnostics() { const auto sim_field_mgr = get_field_manager("sim"); // Create all diagnostics - for (const auto& fname : m_fields_names) { + for (auto& fname : m_fields_names) { if (!sim_field_mgr->has_field(fname)) { - create_diagnostic(fname); - } - } - - // Set required fields for all diagnostics - // NOTE: do this *after* creating all diags: in case the required - // field of certain diagnostics is itself a diagnostic, - // we want to make sure the required ones are all built. - for (const auto& dd : m_diagnostics) { - const auto& diag = dd.second; - for (const auto& req : diag->get_required_field_requests()) { - const auto& req_field = get_field(req.fid.name(),"sim"); - diag->set_required_field(req_field.get_const()); + auto diag = create_diagnostic(fname); + auto diag_fname = diag->get_diagnostic().name(); + m_diagnostics[diag_fname] = diag; + + // Note: the diag field may have a name different from what was used + // in the input file, so update the name with the actual + // diagnostic field name + fname = diag_fname; } - - // Note: this inits with an invalid timestamp. If by any chance we try to - // output the diagnostic without computing it, we'll get an error. - diag->initialize(util::TimeStamp(),RunType::Initial); } } -void AtmosphereOutput:: -create_diagnostic (const std::string& diag_field_name) { +/* ---------------------------------------------------------- */ +std::shared_ptr +AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { auto& diag_factory = AtmosphereDiagnosticFactory::instance(); // Construct a diagnostic by this name ekat::ParameterList params; std::string diag_name; + std::string diag_avg_cnt_name = ""; + + if (diag_field_name.find("_at_")!=std::string::npos) { + // The diagnostic must be one of + // - ${field_name}_at_lev_${N} <- interface fields still use "_lev_" + // - ${field_name}_at_model_bot + // - ${field_name}_at_model_top + // - ${field_name}_at_${M}X + // where M/N are numbers (N integer), X=Pa, hPa, mb, or m + auto tokens = ekat::split(diag_field_name,"_at_"); + EKAT_REQUIRE_MSG (tokens.size()==2, + "Error! Unexpected diagnostic name: " + diag_field_name + "\n"); + + const auto& fname = tokens.front(); + params.set("field_name",fname); + params.set("grid_name",get_field_manager("sim")->get_grid()->name()); + + params.set("vertical_location", tokens[1]); + params.set("mask_value",m_fill_value); - // If the diagnostic is $field@lev$N/$field_bot/$field_top, - // then we need to set some params - auto tokens = ekat::split(diag_field_name,'@'); - auto last = tokens.back(); - - // FieldAtLevel follows convention variable@lev_N (where N is some integer) - // FieldAtPressureLevel follows convention variable@999mb (where 999 is some integer) - auto lev_and_idx = ekat::split(last,'_'); - auto pos = lev_and_idx[0].find_first_not_of("0123456789"); - auto lev_str = lev_and_idx[0].substr(pos); - - if (last=="tom" || last=="bot" || lev_str=="lev") { - // Diagnostic is a horizontal slice at a specific level - diag_name = "FieldAtLevel"; - tokens.pop_back(); - auto fname = ekat::join(tokens,"_"); - // If the field is itself a diagnostic, make sure it's built - if (diag_factory.has_product(fname) and - m_diagnostics.count(fname)==0) { - create_diagnostic(fname); - m_diag_depends_on_diags[diag_field_name].push_back(fname); - } else { - m_diag_depends_on_diags[diag_field_name].resize(0); - } - auto fid = get_field(fname,"sim").get_header().get_identifier(); - params.set("Field Name", fname); - params.set("Grid Name",fid.get_grid_name()); - params.set("Field Layout",fid.get_layout()); - params.set("Field Units",fid.get_units()); - - // If last is bot or top, will simply use that - params.set("Field Level", lev_and_idx.back()); - } else if (lev_str=="mb" || lev_str=="hPa" || lev_str=="Pa") { - // Diagnostic is a horizontal slice at a specific pressure level - diag_name = "FieldAtPressureLevel"; - auto pres_str = lev_and_idx[0].substr(0,pos); - auto pres_units = lev_and_idx[0].substr(pos); - auto pres_level = std::stoi(pres_str); - // Convert pressure level to Pa, the units of pressure in the simulation - if (pres_units=="mb" || pres_units=="hPa") { - pres_level *= 100; - } - tokens.pop_back(); - auto fname = ekat::join(tokens,"_"); - // If the field is itself a diagnostic, make sure it's built - if (diag_factory.has_product(fname) and - m_diagnostics.count(fname)==0) { - create_diagnostic(fname); - m_diag_depends_on_diags[diag_field_name].push_back(fname); + // Conventions on notation (N=any integer): + // FieldAtLevel : var_at_lev_N, var_at_model_top, var_at_model_bot + // FieldAtPressureLevel: var_at_Nx, with x=mb,Pa,hPa + // FieldAtHeight : var_at_Nm + if (tokens[1].find_first_of("0123456789.")==0) { + auto units_start = tokens[1].find_first_not_of("0123456789."); + auto units = tokens[1].substr(units_start); + if (units=="m") { + diag_name = "FieldAtHeight"; + } else if (units=="mb" or units=="Pa" or units=="hPa") { + diag_name = "FieldAtPressureLevel"; + diag_avg_cnt_name = "_" + tokens[1]; // Set avg_cnt tracking for this specific slice + m_track_avg_cnt = true; // If we have pressure slices we need to be tracking the average count. + } else { + EKAT_ERROR_MSG ("Error! Invalid units x for 'field_at_Nx' diagnostic.\n"); + } } else { - m_diag_depends_on_diags[diag_field_name].resize(0); + diag_name = "FieldAtLevel"; } - auto fid = get_field(fname,"sim").get_header().get_identifier(); - params.set("Field Name", fname); - params.set("Grid Name",fid.get_grid_name()); - params.set("Field Layout",fid.get_layout()); - params.set("Field Units",fid.get_units()); - params.set("Field Target Pressure", pres_level); - params.set("mask_value",m_fill_value); + } else if (diag_field_name=="precip_liq_surf_mass_flux" or + diag_field_name=="precip_ice_surf_mass_flux" or + diag_field_name=="precip_total_surf_mass_flux") { + diag_name = "precip_surf_mass_flux"; + // split will return [X, ''], with X being whatever is before '_surf_mass_flux' + auto type = ekat::split(diag_field_name.substr(7),"_surf_mass_flux").front(); + params.set("precip_type",type); + } else if (diag_field_name=="IceWaterPath" or + diag_field_name=="LiqWaterPath" or + diag_field_name=="RainWaterPath" or + diag_field_name=="RimeWaterPath" or + diag_field_name=="VapWaterPath") { + diag_name = "WaterPath"; + // split will return the list [X, ''], with X being whatever is before 'WaterPath' + params.set("Water Kind",ekat::split(diag_field_name,"WaterPath").front()); + } else if (diag_field_name=="MeridionalVapFlux" or + diag_field_name=="ZonalVapFlux") { + diag_name = "VaporFlux"; + // split will return the list [X, ''], with X being whatever is before 'VapFlux' + params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); } else { diag_name = diag_field_name; - m_diag_depends_on_diags[diag_field_name].resize(0); + } + + // These fields are special case of VerticalLayer diagnostic. + // The diagnostics requires the names be given as param value. + if (diag_name == "z_int" or diag_name == "geopotential_int" or + diag_name == "z_mid" or diag_name == "geopotential_mid" or + diag_name == "dz") { + params.set("diag_name", diag_name); } // Create the diagnostic auto diag = diag_factory.create(diag_name,m_comm,params); diag->set_grids(m_grids_manager); - m_diagnostics.emplace(diag_field_name,diag); - // When using remappers with certain diagnostics the get_field command can be called with both the diagnostic - // name as saved inside the diagnostic and with the name as it is given in the output control file. If it is - // the case that these names don't match we add their pairings to the alternate name map. - if (diag->name() != diag_field_name) { - m_fields_alt_name.emplace(diag->name(),diag_field_name); - m_fields_alt_name.emplace(diag_field_name,diag->name()); + + // Add empty entry for this map, so .at(..) always works + auto& deps = m_diag_depends_on_diags[diag->name()]; + + // Initialize the diagnostic + const auto sim_field_mgr = get_field_manager("sim"); + for (const auto& freq : diag->get_required_field_requests()) { + const auto& fname = freq.fid.name(); + if (!sim_field_mgr->has_field(fname)) { + // This diag depends on another diag. Create and init the dependency + if (m_diagnostics.count(fname)==0) { + m_diagnostics[fname] = create_diagnostic(fname); + } + auto dep = m_diagnostics.at(fname); + deps.push_back(fname); + } + diag->set_required_field (get_field(fname,"sim")); + } + diag->initialize(util::TimeStamp(),RunType::Initial); + // If specified, set avg_cnt tracking for this diagnostic. + if (m_add_time_dim && m_track_avg_cnt) { + const auto diag_field = diag->get_diagnostic(); + const auto name = diag_field.name(); + const auto layout = diag_field.get_header().get_identifier().get_layout(); + set_avg_cnt_tracking(name,diag_avg_cnt_name,layout); + } + + return diag; +} + +// Helper function to mark filled points in a specific layout +void AtmosphereOutput:: +update_avg_cnt_view(const Field& field, view_1d_dev& dev_view) { + // If the dev_view_1d is aliasing the field device view (must be Instant output), + // then there's no point in copying from the field's view to dev_view + const auto& name = field.name(); + const auto& layout = m_layouts.at(name); + const auto& dims = layout.dims(); + const auto rank = layout.rank(); + auto data = dev_view.data(); + const bool is_diagnostic = (m_diagnostics.find(name) != m_diagnostics.end()); + const bool is_aliasing_field_view = + m_avg_type==OutputAvgType::Instant && + field.get_header().get_alloc_properties().get_padding()==0 && + field.get_header().get_parent().expired() && + not is_diagnostic; + const auto fill_value = m_fill_value; + if (not is_aliasing_field_view) { + KT::RangePolicy policy(0,layout.size()); + const auto extents = layout.extents(); + switch (rank) { + case 1: + { + // For rank-1 views, we use strided layout, since it helps us + // handling a few more scenarios + auto src_view_1d = field.get_strided_view(); + auto tgt_view_1d = view_Nd_dev<1>(data,dims[0]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { + if (src_view_1d(i)==fill_value) { + tgt_view_1d(i) = 0.0; + } + }); + break; + } + case 2: + { + auto src_view_2d = field.get_view(); + auto tgt_view_2d = view_Nd_dev<2>(data,dims[0],dims[1]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j; + unflatten_idx(idx,extents,i,j); + if (src_view_2d(i,j)==fill_value) { + tgt_view_2d(i,j) = 0.0; + } + }); + break; + } + case 3: + { + auto src_view_3d = field.get_view(); + auto tgt_view_3d = view_Nd_dev<3>(data,dims[0],dims[1],dims[2]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k; + unflatten_idx(idx,extents,i,j,k); + if (src_view_3d(i,j,k)==fill_value) { + tgt_view_3d(i,j,k) = 0.0; + } + }); + break; + } + case 4: + { + auto src_view_4d = field.get_view(); + auto tgt_view_4d = view_Nd_dev<4>(data,dims[0],dims[1],dims[2],dims[3]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l; + unflatten_idx(idx,extents,i,j,k,l); + if (src_view_4d(i,j,k,l)==fill_value) { + tgt_view_4d(i,j,k,l) = 0.0; + } + }); + break; + } + case 5: + { + auto src_view_5d = field.get_view(); + auto tgt_view_5d = view_Nd_dev<5>(data,dims[0],dims[1],dims[2],dims[3],dims[4]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l,m; + unflatten_idx(idx,extents,i,j,k,l,m); + if (src_view_5d(i,j,k,l,m)==fill_value) { + tgt_view_5d(i,j,k,l,m) = 0.0; + } + }); + break; + } + case 6: + { + auto src_view_6d = field.get_view(); + auto tgt_view_6d = view_Nd_dev<6>(data,dims[0],dims[1],dims[2],dims[3],dims[4],dims[5]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l,m,n; + unflatten_idx(idx,extents,i,j,k,l,m,n); + if (src_view_6d(i,j,k,l,m,n)==fill_value) { + tgt_view_6d(i,j,k,l,m,n) = 0.0; + } + }); + break; + } + default: + EKAT_ERROR_MSG ("Error! Field rank (" + std::to_string(rank) + ") not supported by AtmosphereOutput.\n"); + } } } diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index 03c029c20a09..25a3da0d170c 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -124,13 +124,6 @@ class AtmosphereOutput virtual ~AtmosphereOutput () = default; // Constructor - // Note on the last two inputs: - // - is_restart_run: if true, this is a restarted run. This is only importand - // if this output instance is *not* for writing model restart files, - // and *only* for particular choices of "Averaging Type". - // - is_model_restart_output: if true, this Output is for model restart files. - // In this case, we have to also create an "rpointer.atm" file (which - // contains metadata, and is expected by the component coupled) AtmosphereOutput(const ekat::Comm& comm, const ekat::ParameterList& params, const std::shared_ptr& field_mgr, const std::shared_ptr& grids_mgr); @@ -144,8 +137,11 @@ class AtmosphereOutput void restart (const std::string& filename); void init(); void reset_dev_views(); - void setup_output_file (const std::string& filename, const std::string& fp_precision); - void run (const std::string& filename, const bool write, const int nsteps_since_last_output, + void update_avg_cnt_view(const Field&, view_1d_dev& dev_view); + void setup_output_file (const std::string& filename, const std::string& fp_precision, const scorpio::FileMode mode); + void run (const std::string& filename, + const bool output_step, const bool checkpoint_step, + const int nsteps_since_last_output, const bool allow_invalid_fields = false); long long res_dep_memory_footprint () const; @@ -154,6 +150,11 @@ class AtmosphereOutput return m_io_grid; } + // Option to add a logger + void set_logger(const std::shared_ptr& atm_logger) { + m_atm_logger = atm_logger; + } + protected: // Internal functions void set_grid (const std::shared_ptr& grid); @@ -163,14 +164,18 @@ class AtmosphereOutput std::shared_ptr get_field_manager (const std::string& mode) const; void register_dimensions(const std::string& name); - void register_variables(const std::string& filename, const std::string& fp_precision); + void register_variables(const std::string& filename, const std::string& fp_precision, const scorpio::FileMode mode); void set_degrees_of_freedom(const std::string& filename); std::vector get_var_dof_offsets (const FieldLayout& layout); void register_views(); - Field get_field(const std::string& name, const std::string mode) const; + Field get_field(const std::string& name, const std::string& mode) const; void compute_diagnostic (const std::string& name, const bool allow_invalid_fields = false); void set_diagnostics(); - void create_diagnostic (const std::string& diag_name); + std::shared_ptr + create_diagnostic (const std::string& diag_name); + + // Tracking the averaging of any filled values: + void set_avg_cnt_tracking(const std::string& name, const std::string& avg_cnt_suffix, const FieldLayout& layout); // --- Internal variables --- // ekat::Comm m_comm; @@ -187,10 +192,12 @@ class AtmosphereOutput // How to combine multiple snapshots in the output: Instant, Max, Min, Average OutputAvgType m_avg_type; + Real m_avg_coeff_threshold = 0.5; // % of unfilled values required to not just assign value as FillValue // Internal maps to the output fields, how the columns are distributed, the file dimensions and the global ids. std::vector m_fields_names; - std::map m_fields_alt_name; + std::vector m_avg_cnt_names; + std::map m_field_to_avg_cnt_map; std::map m_layouts; std::map m_dofs; std::map> m_dims; @@ -203,13 +210,19 @@ class AtmosphereOutput // NetCDF: Numeric conversion not representable // Also, by default, don't pick max float, to avoid any overflow if the value // is used inside other calculation and/or remap. - float m_fill_value = DEFAULT_FILL_VALUE; + float m_fill_value = constants::DefaultFillValue().value; // Local views of each field to be used for "averaging" output and writing to file. std::map m_host_views_1d; std::map m_dev_views_1d; + std::map m_local_tmp_avg_cnt_views_1d; + std::map m_avg_coeff_views_1d; bool m_add_time_dim; + bool m_track_avg_cnt = false; + + // The logger to be used throughout the ATM to log message + std::shared_ptr m_atm_logger; }; } //namespace scream diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 651cbe95c823..65e5bf030bd5 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -6,7 +6,7 @@ namespace scream { std::string find_filename_in_rpointer ( - const std::string& casename, + const std::string& filename_prefix, const bool model_restart, const ekat::Comm& comm, const util::TimeStamp& run_t0) @@ -22,11 +22,11 @@ std::string find_filename_in_rpointer ( // If the timestamp is in the filename, then the filename ends with "S.nc", // with S being the string representation of the timestamp + auto ts_len = run_t0.to_string().size(); auto extract_ts = [&] (const std::string& line) -> util::TimeStamp { - auto ts_len = run_t0.to_string().size(); auto min_size = ts_len+3; if (line.size()>=min_size) { - auto ts_str = line.substr(line.size()-min_size); + auto ts_str = line.substr(line.size()-min_size,ts_len); auto ts = util::str_to_time_stamp(ts_str); return ts; } else { @@ -34,40 +34,35 @@ std::string find_filename_in_rpointer ( } }; - // Note: keep swallowing line, even after the first match, since we never wipe - // rpointer.atm, so it might contain multiple matches, and we want to pick - // the last one (which is the last restart file that was written). - while (rpointer_file >> line) { + while ((rpointer_file >> line) and not found) { content += line + "\n"; - if (line.find(casename) != std::string::npos && line.find(suffix) != std::string::npos) { - // Extra check: make sure the date in the filename (if present) precedes this->t0. - // If there's no time stamp in one of the filenames, we assume this is some sort of - // unit test, with no time stamp in the filename, and we accept the filename. - auto ts = extract_ts(line); - if (not ts.is_valid() || !(ts<=run_t0) ) { - found = true; - filename = line; - } - } + found = line.find(filename_prefix+suffix) != std::string::npos && + extract_ts(line)==run_t0; + filename = line; } } + int ifound = int(found); comm.broadcast(&ifound,1,0); found = bool(ifound); - broadcast_string(content,comm,comm.root_rank()); - // If the history restart file is not found, it must be because the last - // model restart step coincided with a model output step, in which case - // a restart history file is not written. - // If that's the case, *disable* output restart, by setting - // 'Restart'->'Perform Restart' = false - // in the input parameter list - EKAT_REQUIRE_MSG (found, - "Error! Restart requested, but no restart file found in 'rpointer.atm'.\n" - " restart case name: " + casename + "\n" - " restart file type: " + std::string(model_restart ? "model restart" : "history restart") + "\n" - " rpointer content:\n" + content); + if (not found) { + broadcast_string(content,comm,comm.root_rank()); + + // If the history restart file is not found, it must be because the last + // model restart step coincided with a model output step, in which case + // a restart history file is not written. + // If that's the case, *disable* output restart, by setting + // 'Restart'->'Perform Restart' = false + // in the input parameter list + EKAT_ERROR_MSG ( + "Error! Restart requested, but no restart file found in 'rpointer.atm'.\n" + " restart filename prefix: " + filename_prefix + "\n" + " restart file type: " + std::string(model_restart ? "model restart" : "history restart") + "\n" + " run t0 : " + run_t0.to_string() + "\n" + " rpointer content:\n" + content); + } // Have the root rank communicate the nc filename broadcast_string(filename,comm,comm.root_rank()); diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 7bad4f9e0ac4..e4d803703128 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -1,14 +1,12 @@ #ifndef SCREAM_IO_UTILS_HPP #define SCREAM_IO_UTILS_HPP -#include "share/util/scream_time_stamp.hpp" #include "share/util/scream_time_stamp.hpp" #include "ekat/util/ekat_string_utils.hpp" #include "ekat/mpi/ekat_comm.hpp" #include -#include namespace scream { @@ -21,8 +19,6 @@ enum class OutputAvgType { Invalid }; -constexpr float DEFAULT_FILL_VALUE = std::numeric_limits::max() / 1e5; - inline std::string e2str(const OutputAvgType avg) { using OAT = OutputAvgType; switch (avg) { @@ -115,7 +111,13 @@ struct IOFileSpecs { std::string filename; int num_snapshots_in_file = 0; int max_snapshots_in_file; - bool file_is_full () const { return num_snapshots_in_file==max_snapshots_in_file; } + + // If positive, flush the output file every these many snapshots + int flush_frequency = -1; + + bool file_is_full () const { return num_snapshots_in_file>=max_snapshots_in_file; } + bool file_needs_flush () const { return flush_frequency>0 and num_snapshots_in_file%flush_frequency==0; } + // Adding number of MPI ranks to the filenamea is useful in testing, since we can run // multiple instances of the same test in parallel (with different number of ranks), // without the risk of them overwriting each other output. @@ -126,6 +128,7 @@ struct IOFileSpecs { // Whether this struct refers to a history restart file bool hist_restart_file = false; + }; std::string find_filename_in_rpointer ( diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 4404e331e292..263cb9e8803f 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -3,6 +3,7 @@ #include "share/io/scorpio_input.hpp" #include "share/io/scream_scorpio_interface.hpp" #include "share/util/scream_timing.hpp" +#include "share/scream_config.hpp" #include "ekat/ekat_parameter_list.hpp" #include "ekat/mpi/ekat_comm.hpp" @@ -10,13 +11,12 @@ #include #include +#include +#include namespace scream { -// Local helper functions: -void set_file_header(const std::string& filename); - void OutputManager:: setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const std::shared_ptr& field_mgr, @@ -60,7 +60,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // Output control EKAT_REQUIRE_MSG(m_params.isSublist("output_control"), - "Error! The output control YAML file for " + m_casename + " is missing the sublist 'output_control'"); + "Error! The output control YAML file for " + m_filename_prefix + " is missing the sublist 'output_control'"); auto& out_control_pl = m_params.sublist("output_control"); // Determine which timestamp to use a reference for output frequency. Two options: // 1. use_case_as_start_reference: TRUE - implies we want to calculate frequency from the beginning of the whole simulation, even if this is a restarted run. @@ -80,18 +80,48 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, m_output_control.timestamp_of_last_write = start_ref ? m_case_t0 : m_run_t0; // File specs - m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",-1); - m_output_file_specs.filename_with_mpiranks = out_control_pl.get("MPI Ranks in Filename",false); - m_output_file_specs.save_grid_data = out_control_pl.get("save_grid_data",!m_is_model_restart_output); + constexpr auto large_int = 1000000; + m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",large_int); + m_output_file_specs.filename_with_mpiranks = out_control_pl.get("MPI Ranks in Filename",false); + m_output_file_specs.save_grid_data = out_control_pl.get("save_grid_data",!m_is_model_restart_output); + + // Here, store if PG2 fields will be present in output streams. + // Will be useful if multiple grids are defined (see below). + bool pg2_grid_in_io_streams = false; + const auto& fields_pl = m_params.sublist("Fields"); + for (auto it=fields_pl.sublists_names_cbegin(); it!=fields_pl.sublists_names_cend(); ++it) { + if (*it == "Physics PG2") pg2_grid_in_io_streams = true; + } // For each grid, create a separate output stream. if (field_mgrs.size()==1) { auto output = std::make_shared(m_io_comm,m_params,field_mgrs.begin()->second,grids_mgr); + output->set_logger(m_atm_logger); m_output_streams.push_back(output); } else { - const auto& fields_pl = m_params.sublist("Fields"); for (auto it=fields_pl.sublists_names_cbegin(); it!=fields_pl.sublists_names_cend(); ++it) { const auto& gname = *it; + + // If this is a GLL grid (or IO Grid is GLL) and PG2 fields + // were found above, we must reset the grid COL tag name to + // be "ncol_d" to avoid conflicting lengths with ncol on + // the PG2 grid. + if (pg2_grid_in_io_streams) { + const auto& grid_pl = fields_pl.sublist(gname); + bool reset_ncol_naming = false; + if (gname == "Physics GLL") reset_ncol_naming = true; + if (grid_pl.isParameter("IO Grid Name")) { + if (grid_pl.get("IO Grid Name") == "Physics GLL") { + reset_ncol_naming = true; + } + } + if (reset_ncol_naming) { + grids_mgr-> + get_grid_nonconst(grid_pl.get("IO Grid Name"))-> + reset_field_tag_name(ShortFieldTagsNames::COL,"ncol_d"); + } + } + EKAT_REQUIRE_MSG (grids_mgr->has_grid(gname), "Error! Output requested on grid '" + gname + "', but the grids manager does not store such grid.\n"); @@ -99,6 +129,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, "Error! Output requested on grid '" + gname + "', but no field manager is available for such grid.\n"); auto output = std::make_shared(m_io_comm,m_params,field_mgrs.at(gname),grids_mgr); + output->set_logger(m_atm_logger); m_output_streams.push_back(output); } } @@ -118,13 +149,30 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, std::vector fields; for (const auto& fn : grid.second->get_geometry_data_names()) { const auto& f = grid.second->get_geometry_data(fn); + + if (f.rank()==0) { + // Right now, this only happens for `dx_short`, a single scalar + // coming from iop. Since that scalar is easily recomputed + // upon restart, we can skip this + // NOTE: without this, the code crash while attempting to get a + // pio decomp for this var, since there are no dimensions. + // Perhaps you can explore setting the var as a global att + continue; + } if (use_suffix) { fields.push_back(f.clone(f.name()+"_"+grid.second->m_short_name)); } else { fields.push_back(f.clone()); } } - auto output = std::make_shared(m_io_comm,fields,grid.second); + + // See comment above for ncol naming with 2+ grids + auto grid_nonconst = grid.second->clone(grid.first,true); + if (grid.first == "Physics GLL" && pg2_grid_in_io_streams) { + grid_nonconst->reset_field_tag_name(ShortFieldTagsNames::COL,"ncol_d"); + } + + auto output = std::make_shared(m_io_comm,fields,grid_nonconst); m_geo_data_streams.push_back(output); } } @@ -144,6 +192,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // File specs m_checkpoint_file_specs.max_snapshots_in_file = 1; + m_checkpoint_file_specs.flush_frequency = 1; m_checkpoint_file_specs.filename_with_mpiranks = pl.get("MPI Ranks in Filename",false); m_checkpoint_file_specs.save_grid_data = false; m_checkpoint_file_specs.hist_restart_file = true; @@ -163,7 +212,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // that is different from the filename_prefix of the current output. auto& restart_pl = m_params.sublist("Restart"); bool perform_history_restart = restart_pl.get("Perform Restart",true); - auto hist_restart_casename = restart_pl.get("filename_prefix",m_casename); + auto hist_restart_filename_prefix = restart_pl.get("filename_prefix",m_filename_prefix); if (m_is_model_restart_output) { // For model restart output, the restart time (which is the start time of this run) is precisely @@ -172,23 +221,89 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, m_output_control.nsamples_since_last_write = 0; } else if (perform_history_restart) { using namespace scorpio; - auto fn = find_filename_in_rpointer(hist_restart_casename,false,m_io_comm,m_run_t0); + auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0); // From restart file, get the time of last write, as well as the current size of the avg sample - m_output_control.timestamp_of_last_write = read_timestamp(fn,"last_write"); - m_output_control.nsamples_since_last_write = get_attribute(fn,"num_snapshots_since_last_write"); + m_output_control.timestamp_of_last_write = read_timestamp(rhist_file,"last_write"); + m_output_control.nsamples_since_last_write = get_attribute(rhist_file,"num_snapshots_since_last_write"); + + if (m_avg_type!=OutputAvgType::Instant) { + m_time_bnds.resize(2); + m_time_bnds[0] = m_output_control.timestamp_of_last_write.days_from(m_case_t0); + } // If the type/freq of output needs restart data, we need to restart the streams - const auto has_restart_data = m_avg_type!=OutputAvgType::Instant && m_output_control.frequency>1; - if (has_restart_data && m_output_control.nsamples_since_last_write>0) { + const bool output_every_step = m_output_control.frequency_units=="nsteps" && + m_output_control.frequency==1; + const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant && not output_every_step; + if (has_checkpoint_data && m_output_control.nsamples_since_last_write>0) { for (auto stream : m_output_streams) { - stream->restart(fn); + stream->restart(rhist_file); + } + } + + // We do NOT allow changing output specs across restart. If you do want to change + // any of these, you MUST start a new output stream (e.g., setting 'Perform Restart: false') + auto old_freq = scorpio::get_attribute(rhist_file,"averaging_frequency"); + EKAT_REQUIRE_MSG (old_freq == m_output_control.frequency, + "Error! Cannot change frequency when performing history restart.\n" + " - old freq: " << old_freq << "\n" + " - new freq: " << m_output_control.frequency << "\n"); + auto old_freq_units = scorpio::get_attribute(rhist_file,"averaging_frequency_units"); + EKAT_REQUIRE_MSG (old_freq_units == m_output_control.frequency_units, + "Error! Cannot change frequency units when performing history restart.\n" + " - old freq units: " << old_freq_units << "\n" + " - new freq units: " << m_output_control.frequency_units << "\n"); + auto old_avg_type = scorpio::get_attribute(rhist_file,"averaging_type"); + EKAT_REQUIRE_MSG (old_avg_type == e2str(m_avg_type), + "Error! Cannot change avg type when performing history restart.\n" + " - old avg type: " << old_avg_type + "\n" + " - new avg type: " << e2str(m_avg_type) << "\n"); + auto old_max_snaps = scorpio::get_attribute(rhist_file,"max_snapshots_per_file"); + EKAT_REQUIRE_MSG (old_max_snaps == m_output_file_specs.max_snapshots_in_file, + "Error! Cannot change max snapshots per file when performing history restart.\n" + " - old max snaps: " << old_max_snaps << "\n" + " - new max snaps: " << m_output_file_specs.max_snapshots_in_file << "\n" + "If you *really* want to change the file capacity, you need to force using a new file, setting\n" + " Restart:\n" + " force_new_file: true\n"); + std::string fp_precision = m_params.get("Floating Point Precision"); + auto old_fp_precision = scorpio::get_attribute(rhist_file,"fp_precision"); + EKAT_REQUIRE_MSG (old_fp_precision == fp_precision, + "Error! Cannot change floating point precision when performing history restart.\n" + " - old fp precision: " << old_fp_precision << "\n" + " - new fp precision: " << fp_precision << "\n"); + + // Check if the prev run wrote any output file (it may have not, if the restart was written + // before the 1st output step). If there is a file, check if there's still room in it. + const auto& last_output_filename = get_attribute(rhist_file,"last_output_filename"); + m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); + if (m_resume_output_file) { + scorpio::register_file(last_output_filename,scorpio::Read); + int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); + + // End of checks. Close the file. + scorpio::eam_pio_closefile(last_output_filename); + + // If last output was full, we can no longer try to resume the file + if (num_snaps0, it was already inited during restart + if (m_avg_type!=OutputAvgType::Instant && m_time_bnds.size()==0) { // Init the left hand point of time_bnds based on run/case t0. m_time_bnds.resize(2); m_time_bnds[0] = m_run_t0.days_from(m_case_t0); @@ -216,6 +331,11 @@ void OutputManager::run(const util::TimeStamp& timestamp) if (m_output_disabled) { return; } + + if (m_atm_logger) { + m_atm_logger->debug("[OutputManager::run] filename_prefix: " + m_filename_prefix + "\n"); + } + using namespace scorpio; std::string timer_root = m_is_model_restart_output ? "EAMxx::IO::restart" : "EAMxx::IO::standard"; @@ -229,36 +349,64 @@ void OutputManager::run(const util::TimeStamp& timestamp) // Since we *always* write a history restart file, we can have a non-full checkpoint, if the average // type is Instant and/or the frequency is every step. A non-full checkpoint will simply write some // global attribute, such as the time of last write. + // Also, notice that units="nhours" and freq=1 would still output evey step if dt=3600s. However, + // it is somewhat hard to figure out if output happens every step, without having a dt to compare + // against. Therefore, we simply assume that if units!=nsteps OR freq>1, then we don't output every + // timestep. If, in fact, we are outputing every timestep, it's likely a small test, so it's not too + // bad if we write out some extra data. + const bool output_every_step = m_output_control.frequency_units=="nsteps" && + m_output_control.frequency==1; const bool is_t0_output = timestamp==m_case_t0; const bool is_output_step = m_output_control.is_write_step(timestamp) || is_t0_output; - const bool is_checkpoint_step = m_checkpoint_control.is_write_step(timestamp); - const bool has_checkpoint_data = (m_avg_type!=OutputAvgType::Instant && m_output_control.frequency>1); + const bool is_checkpoint_step = m_checkpoint_control.is_write_step(timestamp) && not is_t0_output; + const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant && not output_every_step; const bool is_full_checkpoint_step = is_checkpoint_step && has_checkpoint_data && not is_output_step; const bool is_write_step = is_output_step || is_checkpoint_step; // Create and setup output/checkpoint file(s), if necessary start_timer(timer_root+"::get_new_file"); - auto setup_output_file = [&](IOControl& control, IOFileSpecs& filespecs, bool add_to_rpointer, const std::string& file_type) { + auto setup_output_file = [&](IOControl& control, IOFileSpecs& filespecs, + bool add_to_rpointer, const std::string& file_type) { // Check if we need to open a new file if (not filespecs.is_open) { + // If this is normal output, with some sort of average, then the timestamp should be + // the one of the last write, since that's when the current avg window started. + // For Instant output (includes model restart output) and history restart, use the current timestamp + auto file_ts = m_avg_type==OutputAvgType::Instant or filespecs.hist_restart_file + ? timestamp : control.timestamp_of_last_write; + + filespecs.filename = compute_filename (control,filespecs,file_ts); // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) - setup_file(filespecs,control,timestamp); + setup_file(filespecs,control); } // If we are going to write an output checkpoint file, or a model restart file, // we need to append to the filename ".rhist" or ".r" respectively, and add // the filename to the rpointer.atm file. - if (add_to_rpointer) { - if (m_io_comm.am_i_root()) { - std::ofstream rpointer; + if (add_to_rpointer && m_io_comm.am_i_root()) { + std::ofstream rpointer; + if (m_is_model_restart_output) { + rpointer.open("rpointer.atm"); // Open rpointer and nuke its content + } else if (is_checkpoint_step) { + // Output restart unit tests do not have a model-output stream that generates rpointer.atm, + // so allow to skip the next check for them. + auto is_unit_testing = m_params.sublist("Checkpoint Control").get("is_unit_testing",false); + EKAT_REQUIRE_MSG (is_unit_testing || std::ifstream("rpointer.atm").good(), + "Error! Cannot find rpointer.atm file to append history restart file in.\n" + " Model restart output is supposed to be in charge of creating rpointer.atm.\n" + " There are two possible causes:\n" + " 1. You have a 'Checkpoint Control' list in your output stream, but no Scorpio::model_restart\n" + " section in the input yaml file. This makes no sense, please correct.\n" + " 2. The current implementation assumes that the model restart OutputManager runs\n" + " *before* any other output stream (so it can nuke rpointer.atm if already existing).\n" + " If this has changed, we need to revisit this piece of the code.\n"); rpointer.open("rpointer.atm",std::ofstream::app); // Open rpointer file and append to it - rpointer << filespecs.filename << std::endl; } + rpointer << filespecs.filename << std::endl; } if (m_atm_logger) { m_atm_logger->info("[EAMxx::output_manager] - Writing " + file_type + ":"); - m_atm_logger->info("[EAMxx::output_manager] CASE: " + m_casename); m_atm_logger->info("[EAMxx::output_manager] FILE: " + filespecs.filename); } }; @@ -284,7 +432,10 @@ void OutputManager::run(const util::TimeStamp& timestamp) const auto& fields_write_filename = is_output_step ? m_output_file_specs.filename : m_checkpoint_file_specs.filename; for (auto& it : m_output_streams) { // Note: filename only matters if is_output_step || is_full_checkpoint_step=true. In that case, it will definitely point to a valid file name. - it->run(fields_write_filename,is_output_step || is_full_checkpoint_step,m_output_control.nsamples_since_last_write,is_t0_output); + if (m_atm_logger) { + m_atm_logger->debug("[OutputManager]: writing fields from grid " + it->get_io_grid()->name() + "...\n"); + } + it->run(fields_write_filename,is_output_step,is_full_checkpoint_step,m_output_control.nsamples_since_last_write,is_t0_output); } stop_timer(timer_root+"::run_output_streams"); @@ -301,13 +452,27 @@ void OutputManager::run(const util::TimeStamp& timestamp) } auto write_global_data = [&](IOControl& control, IOFileSpecs& filespecs) { + if (m_atm_logger) { + m_atm_logger->debug("[OutputManager]: writing globals...\n"); + } if (m_is_model_restart_output) { // Only write nsteps on model restart set_attribute(filespecs.filename,"nsteps",timestamp.get_num_steps()); - } else if (filespecs.hist_restart_file) { - // Update the date of last write and sample size - scorpio::write_timestamp (filespecs.filename,"last_write",m_output_control.timestamp_of_last_write); - scorpio::set_attribute (filespecs.filename,"num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); + } else { + if (filespecs.hist_restart_file) { + // Update the date of last write and sample size + scorpio::write_timestamp (filespecs.filename,"last_write",m_output_control.timestamp_of_last_write); + scorpio::set_attribute (filespecs.filename,"last_output_filename",m_output_file_specs.filename); + scorpio::set_attribute (filespecs.filename,"num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); + } + // Write these in both output and rhist file. The former, b/c we need these info when we postprocess + // output, and the latter b/c we want to make sure these params don't change across restarts + set_attribute(filespecs.filename,"averaging_type",e2str(m_avg_type)); + set_attribute(filespecs.filename,"averaging_frequency_units",m_output_control.frequency_units); + set_attribute(filespecs.filename,"averaging_frequency",m_output_control.frequency); + set_attribute(filespecs.filename,"max_snapshots_per_file",m_output_file_specs.max_snapshots_in_file); + const auto& fp_precision = m_params.get("Floating Point Precision"); + set_attribute(filespecs.filename,"fp_precision",fp_precision); } // Write all stored globals @@ -333,13 +498,15 @@ void OutputManager::run(const util::TimeStamp& timestamp) eam_pio_closefile(filespecs.filename); filespecs.num_snapshots_in_file = 0; filespecs.is_open = false; + } else if (filespecs.file_needs_flush()) { + eam_flush_file (filespecs.filename); } }; start_timer(timer_root+"::update_snapshot_tally"); // Important! Process output file first, and hist restart (if any) second. // That's b/c write_global_data will update m_output_control.timestamp_of_last_write, - // which is later be written as global data in the hist restart file + // which is later written as global data in the hist restart file if (is_output_step) { write_global_data(m_output_control,m_output_file_specs); } @@ -347,7 +514,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) write_global_data(m_checkpoint_control,m_checkpoint_file_specs); } stop_timer(timer_root+"::update_snapshot_tally"); - if (m_time_bnds.size()>0) { + if (is_output_step && m_time_bnds.size()>0) { m_time_bnds[0] = m_time_bnds[1]; } } @@ -382,13 +549,12 @@ long long OutputManager::res_dep_memory_footprint () const { std::string OutputManager:: compute_filename (const IOControl& control, const IOFileSpecs& file_specs, - const bool is_checkpoint_step, const util::TimeStamp& timestamp) const { std::string suffix = - is_checkpoint_step ? ".rhist" + file_specs.hist_restart_file ? ".rhist" : (m_is_model_restart_output ? ".r" : ""); - auto filename = m_casename + suffix; + auto filename = m_filename_prefix + suffix; // Always add avg type and frequency info filename += "." + e2str(m_avg_type); @@ -400,7 +566,7 @@ compute_filename (const IOControl& control, } // Always add a time stamp - if (m_avg_type==OutputAvgType::Instant || is_checkpoint_step) { + if (m_avg_type==OutputAvgType::Instant || file_specs.hist_restart_file) { filename += "." + timestamp.to_string(); } else { filename += "." + control.timestamp_of_last_write.to_string(); @@ -413,10 +579,11 @@ void OutputManager:: set_params (const ekat::ParameterList& params, const std::map>& field_mgrs) { + using vos_t = std::vector; + m_params = params; - if (m_is_model_restart_output) { - using vos_t = std::vector; + if (m_is_model_restart_output) { // We build some restart parameters internally auto avg_type = m_params.get("Averaging Type","INSTANT"); m_avg_type = str2avg(avg_type); @@ -429,6 +596,11 @@ set_params (const ekat::ParameterList& params, "Error! For restart output, max snapshots per file must be 1.\n" " Note: you don't have to specify this parameter for restart output.\n"); + m_output_file_specs.flush_frequency = m_params.get("flush_frequency",1); + EKAT_REQUIRE_MSG (m_output_file_specs.flush_frequency==1, + "Error! For restart output, file flush frequency must be 1.\n" + " Note: you don't have to specify this parameter for restart output.\n"); + auto& fields_pl = m_params.sublist("Fields"); for (const auto& it : field_mgrs) { const auto& fm = it.second; @@ -444,7 +616,7 @@ set_params (const ekat::ParameterList& params, } fields_pl.sublist(it.first).set("Field Names",fnames); } - m_casename = m_params.get("filename_prefix"); + m_filename_prefix = m_params.get("filename_prefix"); // Match precision of Fields m_params.set("Floating Point Precision","real"); } else { @@ -454,134 +626,153 @@ set_params (const ekat::ParameterList& params, "Error! Unsupported averaging type '" + avg_type + "'.\n" " Valid options: Instant, Max, Min, Average. Case insensitive.\n"); - m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",-1); - m_casename = m_params.get("filename_prefix"); + constexpr auto large_int = 1000000; + m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",large_int); + m_filename_prefix = m_params.get("filename_prefix"); + m_output_file_specs.flush_frequency = m_params.get("flush_frequency",m_output_file_specs.max_snapshots_in_file); // Allow user to ask for higher precision for normal model output, // but default to single to save on storage const auto& prec = m_params.get("Floating Point Precision", "single"); - EKAT_REQUIRE_MSG (prec=="single" || prec=="double" || prec=="real", - "Error! Invalid floating point precision '" + prec + "'.\n"); + vos_t valid_prec = {"single", "float", "double", "real"}; + EKAT_REQUIRE_MSG (ekat::contains(valid_prec,prec), + "Error! Invalid/unsupported value for 'Floating Point Precision'.\n" + " - input value: " + prec + "\n" + " - supported values: float, single, double, real\n"); } } /*===============================================================================================*/ void OutputManager:: -setup_file ( IOFileSpecs& filespecs, const IOControl& control, - const util::TimeStamp& timestamp) +setup_file ( IOFileSpecs& filespecs, + const IOControl& control) { using namespace scorpio; const bool is_checkpoint_step = &control==&m_checkpoint_control; - auto& filename = filespecs.filename; - // Compute new file name - filename = compute_filename (control,filespecs,is_checkpoint_step,timestamp); + std::string fp_precision = is_checkpoint_step + ? "real" + : m_params.get("Floating Point Precision"); - // Register new netCDF file for output. First, check no other output managers - // are trying to write on the same file - EKAT_REQUIRE_MSG (not is_file_open_c2f(filename.c_str(),Write), - "Error! File '" + filename + "' is currently open for write. Cannot share with other output managers.\n"); - register_file(filename,Write); + const auto& filename = filespecs.filename; + // Register new netCDF file for output. Check if we need to append to an existing file + auto mode = m_resume_output_file ? Append : Write; + register_file(filename,mode); + if (m_resume_output_file) { + eam_pio_redef(filename); + } - // Note: time has an unknown length. Setting its "length" to 0 tells the scorpio to - // set this dimension as having an 'unlimited' length, thus allowing us to write - // as many timesnaps to file as we desire. + // Note: length=0 is how scorpio recognizes that this is an 'unlimited' dimension, which + // allows to write as many timesnaps as we desire. register_dimension(filename,"time","time",0,false); - // Register time as a variable. + // Register time (and possibly time_bnds) var(s) auto time_units="days since " + m_case_t0.get_date_string() + " " + m_case_t0.get_time_string(); register_variable(filename,"time","time",time_units,{"time"}, "double", "double","time"); -#ifdef SCREAM_HAS_LEAP_YEAR - set_variable_metadata (filename,"time","calendar","gregorian"); -#else - set_variable_metadata (filename,"time","calendar","noleap"); -#endif + if (use_leap_year()) { + set_variable_metadata (filename,"time","calendar","gregorian"); + } else { + set_variable_metadata (filename,"time","calendar","noleap"); + } if (m_avg_type!=OutputAvgType::Instant) { // First, ensure a 'dim2' dimension with len=2 is registered. register_dimension(filename,"dim2","dim2",2,false); - - // Register time_bnds var, with its dofs register_variable(filename,"time_bnds","time_bnds",time_units,{"dim2","time"},"double","double","time-dim2"); - scorpio::offset_t time_bnds_dofs[2] = {0,1}; - set_dof(filename,"time_bnds",2,time_bnds_dofs); - + // Make it clear how the time_bnds should be interpreted - set_variable_metadata(filename,"time_bnds","note","right endpoint accummulation"); + set_variable_metadata(filename,"time_bnds","note","right endpoint accumulation"); // I'm not sure what's the point of this, but CF conventions seem to require it set_variable_metadata (filename,"time","bounds","time_bnds"); } - std::string fp_precision = is_checkpoint_step - ? "real" - : m_params.get("Floating Point Precision"); + if (not m_resume_output_file) { + // Finish the definition phase for this file. + write_timestamp(filename,"case_t0",m_case_t0); + write_timestamp(filename,"run_t0",m_run_t0); + set_attribute(filename,"averaging_type",e2str(m_avg_type)); + set_attribute(filename,"averaging_frequency_units",m_output_control.frequency_units); + set_attribute(filename,"averaging_frequency",m_output_control.frequency); + set_attribute(filename,"max_snapshots_per_file",m_output_file_specs.max_snapshots_in_file); + set_attribute(filename,"fp_precision",fp_precision); + set_file_header(filespecs); + } + + // Set degree of freedom for "time" and "time_bnds" + scorpio::offset_t time_dof[1] = {0}; + set_dof(filename,"time",1,time_dof); + if (m_avg_type!=OutputAvgType::Instant) { + scorpio::offset_t time_bnds_dofs[2] = {0,1}; + set_dof(filename,"time_bnds",2,time_bnds_dofs); + } // Make all output streams register their dims/vars for (auto& it : m_output_streams) { - it->setup_output_file(filename,fp_precision); + it->setup_output_file(filename,fp_precision,mode); } - if (filespecs.save_grid_data) { - // If not a restart file, also register geo data fields. + // If grid data is needed, also register geo data fields. Skip if file is resumed, + // since grid data was written in the previous run + if (filespecs.save_grid_data and not m_resume_output_file) { for (auto& it : m_geo_data_streams) { - it->setup_output_file(filename,fp_precision); + it->setup_output_file(filename,fp_precision,mode); } } - // Set degree of freedom for "time" - scorpio::offset_t time_dof[1] = {0}; - set_dof(filename,"time",0,time_dof); - - // Finish the definition phase for this file. - auto t0_date = m_case_t0.get_date()[0]*10000 + m_case_t0.get_date()[1]*100 + m_case_t0.get_date()[2]; - auto t0_time = m_case_t0.get_time()[0]*10000 + m_case_t0.get_time()[1]*100 + m_case_t0.get_time()[2]; - - set_attribute(filename,"start_date",t0_date); - set_attribute(filename,"start_time",t0_time); - set_attribute(filename,"averaging_type",e2str(m_avg_type)); - set_attribute(filename,"averaging_frequency_units",m_output_control.frequency_units); - set_attribute(filename,"averaging_frequency",m_output_control.frequency); - set_attribute(filename,"max_snapshots_per_file",m_output_file_specs.max_snapshots_in_file); - set_file_header(filename); + // When resuming a file, PIO opens it in data mode. + // NOTE: all the above register_dimension/register_variable are already checking that + // the dims/vars are already in the file (we don't allow adding dims/vars) eam_pio_enddef (filename); - if (m_avg_type!=OutputAvgType::Instant) { - // Unfortunately, attributes cannot be set in define mode (why?), so this could - // not be done while we were setting the time_bnds - set_attribute(filename,"sample_size",control.frequency); - } - - if (filespecs.save_grid_data) { + if (filespecs.save_grid_data and not m_resume_output_file) { // Immediately run the geo data streams for (const auto& it : m_geo_data_streams) { - it->run(filename,true,0); + it->run(filename,true,false,0); } } filespecs.is_open = true; + + m_resume_output_file = false; } /*===============================================================================================*/ -void set_file_header(const std::string& filename) +void OutputManager::set_file_header(const IOFileSpecs& file_specs) { using namespace scorpio; // TODO: All attributes marked TODO below need to be set. Hopefully by a universal value that reflects // what the attribute is. For example, git-hash should be the git-hash associated with this version of // the code at build time for this executable. - set_attribute(filename,"source","E3SM Atmosphere Model Version 4 (EAMxx)"); // TODO: probably want to make sure that new versions are reflected here. - set_attribute(filename,"case",""); // TODO - set_attribute(filename,"title","EAMxx History File"); - set_attribute(filename,"compset",""); // TODO - set_attribute(filename,"git_hash",""); // TODO - set_attribute(filename,"host",""); // TODO - set_attribute(filename,"version",""); // TODO - set_attribute(filename,"initial_file",""); // TODO - set_attribute(filename,"topography_file",""); // TODO - set_attribute(filename,"contact",""); // TODO - set_attribute(filename,"institution_id",""); // TODO - set_attribute(filename,"product",""); // TODO - set_attribute(filename,"component","ATM"); - set_attribute(filename,"Conventions","CF-1.8"); // TODO: In the future we may be able to have this be set at runtime. We hard-code for now, because post-processing needs something in this global attribute. 2023-04-12 + auto& p = m_params.sublist("provenance"); + auto now = std::chrono::system_clock::now(); + std::time_t time = std::chrono::system_clock::to_time_t(now); + std::stringstream timestamp; + timestamp << "created on " << std::ctime(&time); + std::string ts_str = timestamp.str(); + ts_str = std::strtok(&ts_str[0],"\n"); // Remove the \n appended by ctime + + const auto& filename = file_specs.filename; + + set_attribute(filename,"case",p.get("caseid","NONE")); + set_attribute(filename,"source","E3SM Atmosphere Model (EAMxx)"); + set_attribute(filename,"eamxx_version",EAMXX_VERSION); + set_attribute(filename,"git_version",p.get("git_version",EAMXX_GIT_VERSION)); + set_attribute(filename,"hostname",p.get("hostname","UNKNOWN")); + set_attribute(filename,"username",p.get("username","UNKNOWN")); + set_attribute(filename,"atm_initial_conditions_file",p.get("initial_conditions_file","NONE")); + set_attribute(filename,"topography_file",p.get("topography_file","NONE")); + set_attribute(filename,"contact","e3sm-data-support@llnl.gov"); + set_attribute(filename,"institution_id","E3SM-Projet"); + set_attribute(filename,"realm","atmos"); + set_attribute(filename,"history",ts_str); + set_attribute(filename,"Conventions","CF-1.8"); + if (m_is_model_restart_output) { + set_attribute(filename,"product","model-restart"); + } else if (file_specs.hist_restart_file) { + set_attribute(filename,"product","history-restart"); + } else { + set_attribute(filename,"product","model-output"); + } } /*===============================================================================================*/ void OutputManager:: @@ -596,7 +787,7 @@ push_to_logger() }; m_atm_logger->info("[EAMxx::output_manager] - New Output stream"); - m_atm_logger->info(" Case: " + m_casename); + m_atm_logger->info(" Filename prefix: " + m_filename_prefix); m_atm_logger->info(" Run t0: " + m_run_t0.to_string()); m_atm_logger->info(" Case t0: " + m_case_t0.to_string()); m_atm_logger->info(" Reference t0: " + m_output_control.timestamp_of_last_write.to_string()); @@ -608,9 +799,6 @@ push_to_logger() m_atm_logger->info(" Includes Grid Data ?: " + bool_to_string(m_output_file_specs.save_grid_data)); // List each GRID - TODO // List all FIELDS - TODO - - - } } // namespace scream diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index 13027f7153b4..e546922b4d6e 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -118,16 +118,16 @@ class OutputManager std::string compute_filename (const IOControl& control, const IOFileSpecs& file_specs, - const bool is_checkpoint_step, const util::TimeStamp& timestamp) const; + void set_file_header(const IOFileSpecs& file_specs); + // Craft the restart parameter list void set_params (const ekat::ParameterList& params, const std::map>& field_mgrs); void setup_file ( IOFileSpecs& filespecs, - const IOControl& control, - const util::TimeStamp& timestamp); + const IOControl& control); // Manage logging of info to atm.log void push_to_logger(); @@ -144,7 +144,7 @@ class OutputManager ekat::ParameterList m_params; // The output filename root - std::string m_casename; + std::string m_filename_prefix; std::vector m_time_bnds; @@ -168,6 +168,9 @@ class OutputManager // we might have to load an output checkpoint file (depending on avg type) bool m_is_restarted_run; + // Whether a restarted run can resume filling previous run output file (if not full) + bool m_resume_output_file = false; + // If the user specifies freq units "none" or "never", output is disabled bool m_output_disabled = false; diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.F90 b/components/eamxx/src/share/io/scream_scorpio_interface.F90 index 69b3785ddcc8..6f8b83886547 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface.F90 @@ -47,9 +47,6 @@ module scream_scorpio_interface pio_noerr, pio_global, & PIO_int, PIO_real, PIO_double, PIO_float=>PIO_real use pio_kinds, only: PIO_OFFSET_KIND - use pio_nf, only: PIO_enddef, PIO_inq_dimid, PIO_inq_dimlen, PIO_inq_varid, & - PIO_inquire, PIO_inquire_variable - use pionfatt_mod, only: PIO_put_att => put_att use mpi, only: mpi_abort, mpi_comm_size, mpi_comm_rank @@ -60,14 +57,20 @@ module scream_scorpio_interface public :: & lookup_pio_atm_file, & ! Checks if a pio file is present eam_pio_closefile, & ! Close a specfic pio file. - eam_pio_enddef, & ! Register variables and dimensions with PIO files + eam_pio_flush_file, & ! Flushes I/O buffers to file + eam_pio_enddef, & ! Ends define mode phase, enters data mode phase + eam_pio_redef, & ! Pause data mode phase, re-enter define mode phase eam_init_pio_subsystem, & ! Gather pio specific data from the component coupler is_eam_pio_subsystem_inited, & ! Query whether the pio subsystem is inited already eam_pio_finalize, & ! Run any final PIO commands register_file, & ! Creates/opens a pio input/output file register_variable, & ! Register a variable with a particular pio output file - set_variable_metadata, & ! Sets a variable metadata (always char data) - get_variable, & ! Register a variable with a particular pio output file + set_variable_metadata_char, & ! Sets a variable metadata (char data) + set_variable_metadata_float, & ! Sets a variable metadata (float data) + set_variable_metadata_double,& ! Sets a variable metadata (double data) + get_variable_metadata_char, & ! Gets a variable metadata (char data) + get_variable_metadata_float, & ! Gets a variable metadata (float data) + get_variable_metadata_double,& ! Gets a variable metadata (double data) register_dimension, & ! Register a dimension with a particular pio output file set_decomp, & ! Set the pio decomposition for all variables in file. set_dof, & ! Set the pio dof decomposition for specific variable in file. @@ -76,7 +79,7 @@ module scream_scorpio_interface eam_update_time, & ! Update the timestamp (i.e. time variable) for a given pio netCDF file read_time_at_index ! Returns the time stamp for a specific time index - private :: errorHandle, get_coord + private :: errorHandle, get_coord, is_read, is_write, is_append ! Universal PIO variables for the module integer :: atm_mpicom @@ -91,17 +94,18 @@ module scream_scorpio_interface integer,parameter :: max_chars = 256 integer,parameter :: max_hvarname_len = 64 integer,parameter :: max_hvar_dimlen = 5 - integer,parameter :: file_purpose_not_set = 0 - integer,parameter :: file_purpose_in = 1 - integer,parameter :: file_purpose_out = 2 + integer,parameter :: file_purpose_not_set = 0 ! Not set (to catch uninited stuff errors) + integer,parameter :: file_purpose_in = 1 ! Read + integer,parameter :: file_purpose_app = 2 ! Write (append if file exists) + integer,parameter :: file_purpose_out = 4 ! Write (replace if file exists) type, public :: hist_coord_t character(len=max_hcoordname_len) :: name = '' ! coordinate name - integer :: dimsize = 0 ! size of dimension integer :: dimid ! Unique PIO Id for this dimension character(len=max_chars) :: long_name = '' ! 'long_name' attribute character(len=max_chars) :: units = '' ! 'units' attribute logical :: is_partitioned ! whether the dimension is partitioned across ranks + logical :: is_time_dim ! whether the dimension is the time (unlimited) dim end type hist_coord_t type, public :: hist_var_t @@ -113,7 +117,7 @@ module scream_scorpio_interface integer :: dtype ! data type used to pass data to read/write routines integer :: nc_dtype ! data type used in the netcdf files integer :: numdims ! Number of dimensions in out field - type(io_desc_t), pointer :: iodesc ! PIO decomp associated with this variable + type(io_desc_t), pointer :: iodesc => NULL() ! PIO decomp associated with this variable type(iodesc_list_t), pointer :: iodesc_list ! PIO decomp list with metadata about PIO decomp integer(kind=pio_offset_kind), allocatable :: compdof(:) ! Global locations in output array for this process integer, allocatable :: dimid(:) ! array of PIO dimension id's for this variable @@ -207,6 +211,7 @@ end subroutine register_file ! of a new PIO file. Once this routine is called it is not possible ! to add new dimensions or variables to the file. subroutine eam_pio_enddef(filename) + use pio, only: PIO_enddef character(len=*), intent(in) :: filename @@ -222,7 +227,6 @@ subroutine eam_pio_enddef(filename) ! It could happen that we are running a test, with an input file opening the ! same file that an output stream just wrote. In this case, the def phase ended ! during the output setup. - ! if (.not. current_atm_file%is_enddef) then ! Gather the pio decomposition for all variables in this file, and assign them pointers. call set_decomp(trim(filename)) @@ -233,6 +237,35 @@ subroutine eam_pio_enddef(filename) endif end subroutine eam_pio_enddef +!=====================================================================! + ! Mandatory call to finish the variable and dimension definition phase + ! of a new PIO file. Once this routine is called it is not possible + ! to add new dimensions or variables to the file. + subroutine eam_pio_redef(filename) + use pio, only: pio_redef + + character(len=*), intent(in) :: filename + + type(pio_atm_file_t), pointer :: current_atm_file + integer :: ierr + logical :: found + + call lookup_pio_atm_file(filename,current_atm_file,found) + if (.not.found) then + call errorHandle("PIO ERROR: error running redef on file "//trim(filename)//".\n PIO file not found or not open.",-999) + endif + + ! It could happen that we are running a test, with an input file opening the + ! same file that an output stream just wrote. In this case, the def phase ended + ! during the output setup. + if (current_atm_file%is_enddef) then + ! Re-open define phase + ierr = PIO_redef(current_atm_file%pioFileDesc) + call errorHandle("PIO ERROR: issue arose with PIO_redef for file"//trim(current_atm_file%filename),ierr) + current_atm_file%is_enddef = .false. + endif + + end subroutine eam_pio_redef !=====================================================================! ! Register a dimension with a specific pio output file. Mandatory inputs ! include: @@ -244,9 +277,11 @@ end subroutine eam_pio_enddef ! length: The dimension length (must be >=0). Choosing 0 marks the ! dimensions as having "unlimited" length which is used for ! dimensions such as time. + ! NOTE: if the file is in Read or Append mode, we are supposed to have already checked + ! that the specs match the ones in the file during the C++ wrapper functions subroutine register_dimension(filename,shortname,longname,length,is_partitioned) use pio_types, only: pio_unlimited - use pio_nf, only: PIO_def_dim + use pio, only: PIO_def_dim character(len=*), intent(in) :: filename ! Name of file to register the dimension on. character(len=*), intent(in) :: shortname,longname ! Short- and long- names for this dimension, short: brief identifier and name for netCDF output, long: longer descriptor sentence to be included as meta-data in file. @@ -282,6 +317,7 @@ subroutine register_dimension(filename,shortname,longname,length,is_partitioned) prev => curr curr => prev%next end do + ! If the dim was not found, create it if (.not. dim_found) then allocate(prev%next) @@ -292,32 +328,25 @@ subroutine register_dimension(filename,shortname,longname,length,is_partitioned) ! Register this dimension hist_coord%name = trim(shortname) hist_coord%long_name = trim(longname) - hist_coord%dimsize = length hist_coord%is_partitioned = is_partitioned + hist_coord%is_time_dim = length .eq. 0 - if (length.eq.0) then - ierr = PIO_def_dim(pio_atm_file%pioFileDesc, trim(shortname), pio_unlimited , hist_coord%dimid) - time_dimid = hist_coord%dimid - else - ierr = PIO_def_dim(pio_atm_file%pioFileDesc, trim(shortname), length , hist_coord%dimid) - end if - call errorHandle("PIO ERROR: could not define dimension "//trim(shortname)//" on file: "//trim(filename),ierr) - else - ! The dim was already registered by another input/output instance. Check that everything matches - hist_coord => curr%coord - if (trim(hist_coord%name) .ne. trim(shortname) .or. & - trim(hist_coord%long_name) .ne. trim(longname) .or. & - hist_coord%dimsize .ne. length) then - call errorHandle("PIO ERROR: dimension "//trim(shortname)//" was already defined on file "//trim(filename)//", but with a different length",ierr) - + if (is_write(pio_atm_file%purpose)) then + if (length.eq.0) then + ierr = PIO_def_dim(pio_atm_file%pioFileDesc, trim(shortname), pio_unlimited , hist_coord%dimid) + time_dimid = hist_coord%dimid + else + ierr = PIO_def_dim(pio_atm_file%pioFileDesc, trim(shortname), length , hist_coord%dimid) + end if + call errorHandle("PIO ERROR: could not define dimension "//trim(shortname)//" on file: "//trim(filename),ierr) endif endif end subroutine register_dimension !=====================================================================! - ! Register a variable with a specific pio input file. Mandatory inputs + ! Register a variable with a specific pio input/output file. Mandatory inputs ! include: - ! filename: The name of the netCDF file this variable will be - ! registered with. + ! pio_atm_filename: The name of the netCDF file this variable will be + ! registered with. ! shortname: A shortname descriptor (tag) for this variable. This will be ! used to label the variable in the netCDF file as well. ! longname: A longer character string describing the variable. @@ -333,21 +362,29 @@ end subroutine register_dimension ! decomposition for reading this variable. It is ok to reuse ! the pio_decomp_tag for variables that have the same ! dimensionality. See get_decomp for more details. - subroutine get_variable(filename,shortname,longname,numdims,var_dimensions,dtype,pio_decomp_tag) - use pio_nf, only: PIO_inq_vartype - character(len=*), intent(in) :: filename ! Name of the file to register this variable with - character(len=*), intent(in) :: shortname,longname ! short and long names for the variable. Short: variable name in file, Long: more descriptive name - integer, intent(in) :: numdims ! Number of dimensions for this variable, including time dimension - character(len=*), intent(in) :: var_dimensions(numdims) ! String array with shortname descriptors for each dimension of variable. - integer, intent(in) :: dtype ! datatype for this variable, REAL, DOUBLE, INTEGER, etc. - character(len=*), intent(in) :: pio_decomp_tag ! Unique tag for this variables decomposition type, to be used to determine if the io-decomp already exists. + ! NOTE: if the file is in Read or Append mode, we are supposed to have already checked + ! that the specs match the ones in the file during the C++ wrapper functions + subroutine register_variable(filename,shortname,longname,units, & + numdims,var_dimensions, & + dtype,nc_dtype,pio_decomp_tag) + use pio, only: PIO_def_var, PIO_inq_dimid, PIO_inq_dimlen, PIO_inq_varid, PIO_put_att + + character(len=256), intent(in) :: filename ! Name of the file to register this variable with + character(len=256), intent(in) :: shortname,longname ! short and long names for the variable. Short: variable name in file, Long: more descriptive name + character(len=256), intent(in) :: units ! units for variable + integer, intent(in) :: numdims ! Number of dimensions for this variable, including time dimension + character(len=256), intent(in) :: var_dimensions(numdims) ! String array with shortname descriptors for each dimension of variable. + character(len=256), intent(in) :: pio_decomp_tag ! Unique tag for this variables decomposition type, to be used to determine if the io-decomp already exists. + integer, intent(in) :: dtype ! datatype for arrays that will be passed to read/write routines + integer, intent(in) :: nc_dtype ! datatype for this variable in nc files (unused if file mode is Read) ! Local variables type(pio_atm_file_t),pointer :: pio_atm_file type(hist_var_t), pointer :: hist_var integer :: dim_ii - integer :: ierr logical :: found,var_found + integer :: ierr + type(hist_coord_t), pointer :: hist_coord type(hist_var_list_t), pointer :: curr, prev @@ -362,233 +399,265 @@ subroutine get_variable(filename,shortname,longname,numdims,var_dimensions,dtype ! Get a new variable pointer in var_list if (len_trim(shortname)>max_hvarname_len) call errorHandle("PIO Error: variable shortname "//trim(shortname)//" is too long, consider increasing max_hvarname_len or changing the variable shortname",-999) curr => pio_atm_file%var_list_top - - do while (associated(curr)) + do while ( associated(curr) ) if (associated(curr%var)) then if (trim(curr%var%name)==trim(shortname) .and. curr%var%is_set) then - var_found = .true. exit - endif - endif + end if + end if prev => curr curr => prev%next end do + + allocate(prev%next) + curr => prev%next + allocate(curr%var) + hist_var => curr%var + ! Populate meta-data associated with this variable + hist_var%name = trim(shortname) + hist_var%long_name = trim(longname) + + hist_var%units = trim(units) + hist_var%numdims = numdims + hist_var%dtype = dtype + hist_var%nc_dtype = nc_dtype + hist_var%pio_decomp_tag = trim(pio_decomp_tag) + ! Determine the dimension id's saved in the netCDF file and associated with + ! this variable, check if variable has a time dimension + hist_var%has_t_dim = .false. + hist_var%is_partitioned = .false. + allocate(hist_var%dimid(numdims),hist_var%dimlen(numdims)) + do dim_ii = 1,numdims + ierr = pio_inq_dimid(pio_atm_file%pioFileDesc,trim(var_dimensions(dim_ii)),hist_var%dimid(dim_ii)) + call errorHandle("EAM_PIO ERROR: Unable to find dimension id for "//trim(var_dimensions(dim_ii)),ierr) + ierr = pio_inq_dimlen(pio_atm_file%pioFileDesc,hist_var%dimid(dim_ii),hist_var%dimlen(dim_ii)) + call errorHandle("EAM_PIO ERROR: Unable to determine length for dimension "//trim(var_dimensions(dim_ii)),ierr) + + call get_coord (filename,var_dimensions(dim_ii),hist_coord) + if (hist_coord%is_partitioned) then + hist_var%is_partitioned = .true. + endif + if (hist_coord%is_time_dim) then + hist_var%has_t_dim = .true. + endif + end do - if (.not. var_found) then - allocate(prev%next) - curr => prev%next - allocate(curr%var) - hist_var => curr%var - ! Populate meta-data associated with this variable - hist_var%name = trim(shortname) - hist_var%long_name = trim(longname) - hist_var%numdims = numdims - hist_var%dtype = dtype - hist_var%pio_decomp_tag = trim(pio_decomp_tag) - ! Determine the dimension id's saved in the netCDF file and associated with - ! this variable, check if variable has a time dimension - hist_var%has_t_dim = .false. - hist_var%is_partitioned = .false. - allocate(hist_var%dimid(numdims),hist_var%dimlen(numdims)) - do dim_ii = 1,numdims - ierr = pio_inq_dimid(pio_atm_file%pioFileDesc,trim(var_dimensions(dim_ii)),hist_var%dimid(dim_ii)) - call errorHandle("EAM_PIO ERROR: Unable to find dimension id for "//trim(var_dimensions(dim_ii)),ierr) - ierr = pio_inq_dimlen(pio_atm_file%pioFileDesc,hist_var%dimid(dim_ii),hist_var%dimlen(dim_ii)) - call errorHandle("EAM_PIO ERROR: Unable to determine length for dimension "//trim(var_dimensions(dim_ii)),ierr) - if (hist_var%dimlen(dim_ii).eq.0) hist_var%has_t_dim = .true. - end do - - ! Register Variable with PIO - ! check to see if variable already is defined with file (for use with input) + if (is_write(pio_atm_file%purpose)) then + ierr = PIO_def_var(pio_atm_file%pioFileDesc, trim(shortname), hist_var%nc_dtype, hist_var%dimid(:numdims), hist_var%piovar) + call errorHandle("PIO ERROR: could not define variable "//trim(shortname),ierr) + ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'units', hist_var%units ) + ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'long_name', hist_var%long_name ) + else ierr = PIO_inq_varid(pio_atm_file%pioFileDesc,trim(shortname),hist_var%piovar) - call errorHandle("PIO ERROR: could not find variable "//trim(shortname)//" in file "//trim(filename),ierr) + call errorHandle("PIO ERROR: could not retrieve id for variable "//trim(shortname)//" from file "//trim(filename),ierr) + endif - ! Not really needed, but just in case, store var data type in the nc file - ierr = PIO_inq_vartype(pio_atm_file%pioFileDesc,hist_var%piovar,hist_var%nc_dtype) - call errorHandle("EAM PIO ERROR: Unable to retrieve dtype for variable "//shortname,ierr) + ! Set that new variable has been created + hist_var%is_set = .true. - ! Set that the new variable has been set - hist_var%is_set = .true. - else - ! The var was already registered by another input/output instance. Check that everything matches - hist_var => curr%var - if ( trim(hist_var%long_name) .ne. trim(longname) ) then - ! Different long name - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different longname, in file: "//trim(filename),-999) - elseif (hist_var%dtype .ne. dtype) then - ! Different data type - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different dtype, in file: "//trim(filename),-999) - elseif (pio_atm_file%purpose .eq. file_purpose_out .and. & ! Out files must match the decomp tag - (hist_var%numdims .ne. numdims .or. & - trim(hist_var%pio_decomp_tag) .ne. trim(pio_decomp_tag))) then - ! Different decomp tag in output file - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different decomp tag, in file: "//trim(filename),-999) - elseif (hist_var%numdims .ne. numdims .and. & - hist_var%numdims .ne. (numdims+1)) then - ! Invalid dimlen - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different dimlen, in file: "//trim(filename),-999) - elseif (pio_atm_file%purpose .eq. file_purpose_in .and. & - hist_var%numdims .eq. numdims .and. & - trim(hist_var%pio_decomp_tag) .ne. trim(pio_decomp_tag)) then - ! Same dimlen, but different decomp tag in input file - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different decomp tag, in file: "//trim(filename),-999) - elseif (pio_atm_file%purpose .eq. file_purpose_in .and. & ! In files *may* use a decomp tag - (hist_var%numdims .eq. (numdims+1) .and. & ! without "-time" at the end - trim(hist_var%pio_decomp_tag) .ne. trim(pio_decomp_tag)//"-time")) then - ! Different dimlen, but different decomp tag even if attaching "-time" in input file - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different decomp tag, in file: "//trim(filename),-999) + end subroutine register_variable +!=====================================================================! + subroutine set_variable_metadata_float(filename, varname, metaname, metaval) + use pio, only: PIO_put_att + + character(len=256), intent(in) :: filename + character(len=256), intent(in) :: varname + character(len=256), intent(in) :: metaname + real(kind=c_float), intent(in) :: metaval + + ! Local variables + type(pio_atm_file_t),pointer :: pio_file + type(hist_var_t), pointer :: var + integer :: ierr + logical :: found + + type(hist_var_list_t), pointer :: curr + + ! Find the pointer for this file + call lookup_pio_atm_file(trim(filename),pio_file,found) + if (.not.found ) then + call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) + endif + + ! Find the variable in the file + curr => pio_file%var_list_top + + found = .false. + do while (associated(curr)) + if (associated(curr%var)) then + if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then + found = .true. + var => curr%var + exit + endif endif + curr => curr%next + end do + if (.not.found ) then + call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n Variable not found.",-999) endif - end subroutine get_variable + + ierr = PIO_put_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) + if (ierr .ne. 0) then + call errorHandle("Error setting attribute '" // trim(metaname) & + // "' on variable '" // trim(varname) & + // "' in pio file " // trim(filename) // ".", -999) + endif + + end subroutine set_variable_metadata_float !=====================================================================! - ! Register a variable with a specific pio output file. Mandatory inputs - ! include: - ! pio_atm_filename: The name of the netCDF file this variable will be - ! registered with. - ! shortname: A shortname descriptor (tag) for this variable. This will be - ! used to label the variable in the netCDF file as well. - ! longname: A longer character string describing the variable. - ! numdims: The number of dimensions associated with this variable, - ! including time (if applicable). - ! var_dimensions: An array of character strings with the dimension shortnames - ! for each dimension used by this variable. Should have - ! 'numdims' entries. - ! dtype: The data type for this variable using the proper netCDF - ! integer tag. - ! pio_decomp_tag: A string that describes this particular dimension - ! arrangement which will be used to create a unique PIO - ! decomposition for reading this variable. It is ok to reuse - ! the pio_decomp_tag for variables that have the same - ! dimensionality. See get_decomp for more details. - subroutine register_variable(filename,shortname,longname,units, & - numdims,var_dimensions, & - dtype,nc_dtype,pio_decomp_tag) - use pio_nf, only: PIO_def_var + subroutine set_variable_metadata_double(filename, varname, metaname, metaval) + use pio, only: PIO_put_att - character(len=*), intent(in) :: filename ! Name of the file to register this variable with - character(len=*), intent(in) :: shortname,longname ! short and long names for the variable. Short: variable name in file, Long: more descriptive name - character(len=*), intent(in) :: units ! units for variable - integer, intent(in) :: numdims ! Number of dimensions for this variable, including time dimension - character(len=*), intent(in) :: var_dimensions(numdims) ! String array with shortname descriptors for each dimension of variable. - integer, intent(in) :: dtype ! datatype for arrays that will be passed to read/write routines - integer, intent(in) :: nc_dtype ! datatype for this variable in nc files - character(len=*), intent(in) :: pio_decomp_tag ! Unique tag for this variables decomposition type, to be used to determine if the io-decomp already exists. + character(len=256), intent(in) :: filename + character(len=256), intent(in) :: varname + character(len=256), intent(in) :: metaname + real(kind=c_double), intent(in) :: metaval ! Local variables - type(pio_atm_file_t),pointer :: pio_atm_file - type(hist_var_t), pointer :: hist_var - integer :: dim_ii - logical :: found,var_found + type(pio_atm_file_t),pointer :: pio_file + type(hist_var_t), pointer :: var integer :: ierr - character(len=256) :: dimlen_str - type(hist_coord_t), pointer :: hist_coord + logical :: found - type(hist_var_list_t), pointer :: curr, prev - - var_found = .false. + type(hist_var_list_t), pointer :: curr ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) + call lookup_pio_atm_file(trim(filename),pio_file,found) if (.not.found ) then - call errorHandle("PIO ERROR: error registering variable "//trim(shortname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) + call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) endif - ! Get a new variable pointer in var_list - if (len_trim(shortname)>max_hvarname_len) call errorHandle("PIO Error: variable shortname "//trim(shortname)//" is too long, consider increasing max_hvarname_len or changing the variable shortname",-999) - curr => pio_atm_file%var_list_top + ! Find the variable in the file + curr => pio_file%var_list_top + found = .false. do while (associated(curr)) if (associated(curr%var)) then - if (trim(curr%var%name)==trim(shortname) .and. curr%var%is_set) then - var_found = .true. + if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then + found = .true. + var => curr%var exit endif - end if - prev => curr - curr => prev%next + endif + curr => curr%next end do + if (.not.found ) then + call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n Variable not found.",-999) + endif - ! If the var was not found, allocate the new var - if (.not. var_found) then - allocate(prev%next) - curr => prev%next - allocate(curr%var) - hist_var => curr%var - ! Populate meta-data associated with this variable - hist_var%name = trim(shortname) - hist_var%long_name = trim(longname) - hist_var%units = trim(units) - hist_var%numdims = numdims - hist_var%dtype = dtype - hist_var%nc_dtype = nc_dtype - hist_var%pio_decomp_tag = trim(pio_decomp_tag) - ! Determine the dimension id's saved in the netCDF file and associated with - ! this variable, check if variable has a time dimension - hist_var%has_t_dim = .false. - hist_var%is_partitioned = .false. - allocate(hist_var%dimid(numdims),hist_var%dimlen(numdims)) - do dim_ii = 1,numdims - ierr = pio_inq_dimid(pio_atm_file%pioFileDesc,trim(var_dimensions(dim_ii)),hist_var%dimid(dim_ii)) - call errorHandle("EAM_PIO ERROR: Unable to find dimension id for "//trim(var_dimensions(dim_ii)),ierr) - ierr = pio_inq_dimlen(pio_atm_file%pioFileDesc,hist_var%dimid(dim_ii),hist_var%dimlen(dim_ii)) - call errorHandle("EAM_PIO ERROR: Unable to determine length for dimension "//trim(var_dimensions(dim_ii)),ierr) - if (hist_var%dimlen(dim_ii).eq.0) hist_var%has_t_dim = .true. - call convert_int_2_str(hist_var%dimlen(dim_ii),dimlen_str) - hist_var%pio_decomp_tag = hist_var%pio_decomp_tag//"_"//trim(dimlen_str) - - call get_coord (filename,var_dimensions(dim_ii),hist_coord) - if (hist_coord%is_partitioned) then - hist_var%is_partitioned = .true. + ierr = PIO_put_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) + if (ierr .ne. 0) then + call errorHandle("Error setting attribute '" // trim(metaname) & + // "' on variable '" // trim(varname) & + // "' in pio file " // trim(filename) // ".", -999) + endif + + end subroutine set_variable_metadata_double +!=====================================================================! + function get_variable_metadata_float(filename, varname, metaname) result(metaval) + use pio, only: PIO_get_att + + character(len=256), intent(in) :: filename + character(len=256), intent(in) :: varname + character(len=256), intent(in) :: metaname + real(kind=c_float) :: metaval + + + ! Local variables + type(pio_atm_file_t),pointer :: pio_file + type(hist_var_t), pointer :: var + integer :: ierr + logical :: found + + type(hist_var_list_t), pointer :: curr + + ! Find the pointer for this file + call lookup_pio_atm_file(trim(filename),pio_file,found) + if (.not.found ) then + call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) + endif + + ! Find the variable in the file + curr => pio_file%var_list_top + + found = .false. + do while (associated(curr)) + if (associated(curr%var)) then + if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then + found = .true. + var => curr%var + exit endif - end do + endif + curr => curr%next + end do + if (.not.found ) then + call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) + endif - ierr = PIO_def_var(pio_atm_file%pioFileDesc, trim(shortname), hist_var%nc_dtype, hist_var%dimid(:numdims), hist_var%piovar) - call errorHandle("PIO ERROR: could not define variable "//trim(shortname),ierr) + ! TODO: Maybe we shouldn't throw an error when the metadata isn't there, maybe we provide an output like ierr as an integer? + ! That way we can query for a metadata and on the EAMxx side decide what to do if the metadata is missing. + ierr = PIO_get_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) + if (ierr .ne. 0) then + call errorHandle("Error getting attribute '" // trim(metaname) & + // "' on variable '" // trim(varname) & + // "' in pio file " // trim(filename) // ".", -999) + endif - !PMC - ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'units', hist_var%units ) - ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'long_name', hist_var%long_name ) + end function get_variable_metadata_float +!=====================================================================! + function get_variable_metadata_double(filename, varname, metaname) result(metaval) + use pio, only: PIO_get_att - ! Set that new variable has been created - hist_var%is_set = .true. - else - ! The var was already registered by another input/output instance. Check that everything matches - hist_var => curr%var - if ( trim(hist_var%long_name) .ne. trim(longname) ) then - ! Different long name - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different longname, in file: "//trim(filename),-999) - elseif ( trim(hist_var%units) .ne. trim(units) ) then - ! Different units - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different units, in file: "//trim(filename),-999) - elseif (hist_var%nc_dtype .ne. nc_dtype .or. hist_var%dtype .ne. dtype) then - ! Different data type - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different dtype, in file: "//trim(filename),-999) - elseif (pio_atm_file%purpose .eq. file_purpose_out .and. & ! Out files must match the decomp tag - (hist_var%numdims .ne. numdims .or. & - trim(hist_var%pio_decomp_tag) .ne. trim(pio_decomp_tag))) then - ! Different decomp tag in output file - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different decomp tag, in file: "//trim(filename),-999) - elseif (hist_var%numdims .ne. numdims .and. & - hist_var%numdims .ne. (numdims+1)) then - ! Invalid dimlen - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different dimlen, in file: "//trim(filename),-999) - elseif (pio_atm_file%purpose .eq. file_purpose_in .and. & - hist_var%numdims .eq. numdims .and. & - trim(hist_var%pio_decomp_tag) .ne. trim(pio_decomp_tag)) then - ! Same dimlen, but different decomp tag in input file - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different decomp tag, in file: "//trim(filename),-999) - elseif (pio_atm_file%purpose .eq. file_purpose_in .and. & ! In files *may* use a decomp tag - (hist_var%numdims .eq. (numdims+1) .and. & ! without "-time" at the end - trim(hist_var%pio_decomp_tag) .ne. trim(pio_decomp_tag)//"-time")) then - ! Different dimlen, but different decomp tag even if attaching "-time" in input file - call errorHandle("PIO Error: variable "//trim(shortname)//", already registered with different decomp tag, in file: "//trim(filename),-999) + character(len=256), intent(in) :: filename + character(len=256), intent(in) :: varname + character(len=256), intent(in) :: metaname + real(kind=c_double) :: metaval + + ! Local variables + type(pio_atm_file_t),pointer :: pio_file + type(hist_var_t), pointer :: var + integer :: ierr + logical :: found + + type(hist_var_list_t), pointer :: curr + + ! Find the pointer for this file + call lookup_pio_atm_file(trim(filename),pio_file,found) + if (.not.found ) then + call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) + endif + + ! Find the variable in the file + curr => pio_file%var_list_top + + found = .false. + do while (associated(curr)) + if (associated(curr%var)) then + if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then + found = .true. + var => curr%var + exit + endif endif + curr => curr%next + end do + if (.not.found ) then + call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) endif - end subroutine register_variable + ierr = PIO_get_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) + if (ierr .ne. 0) then + call errorHandle("Error getting attribute '" // trim(metaname) & + // "' on variable '" // trim(varname) & + // "' in pio file " // trim(filename) // ".", -999) + endif + + end function get_variable_metadata_double !=====================================================================! - subroutine set_variable_metadata(filename, varname, metaname, metaval) - use pionfatt_mod, only: PIO_put_att => put_att + subroutine set_variable_metadata_char(filename, varname, metaname, metaval) + use pio, only: PIO_put_att character(len=256), intent(in) :: filename character(len=256), intent(in) :: varname @@ -634,7 +703,57 @@ subroutine set_variable_metadata(filename, varname, metaname, metaval) // "' in pio file " // trim(filename) // ".", -999) endif - end subroutine set_variable_metadata + end subroutine set_variable_metadata_char +!=====================================================================! + function get_variable_metadata_char(filename, varname, metaname) result(metaval) + use pio, only: PIO_get_att + + character(len=256), intent(in) :: filename + character(len=256), intent(in) :: varname + character(len=256), intent(in) :: metaname + character(len=256) :: metaval + + ! Local variables + type(pio_atm_file_t),pointer :: pio_file + type(hist_var_t), pointer :: var + integer :: ierr + logical :: found + + type(hist_var_list_t), pointer :: curr + + ! Find the pointer for this file + call lookup_pio_atm_file(trim(filename),pio_file,found) + if (.not.found ) then + call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) + endif + + ! Find the variable in the file + curr => pio_file%var_list_top + + found = .false. + do while (associated(curr)) + if (associated(curr%var)) then + if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then + found = .true. + var => curr%var + exit + endif + endif + curr => curr%next + end do + if (.not.found ) then + call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) + endif + + ! TODO: Maybe we shouldn't throw an error when the metadata isn't there, maybe we provide an output like ierr as an integer? + ! That way we can query for a metadata and on the EAMxx side decide what to do if the metadata is missing. + ierr = PIO_get_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) + if (ierr .ne. 0) then + call errorHandle("Error getting attribute '" // trim(metaname) & + // "' on variable '" // trim(varname) & + // "' in pio file " // trim(filename) // ".", -999) + endif + end function get_variable_metadata_char !=====================================================================! ! Update the time dimension for a specific PIO file. This is needed when ! reading or writing multiple time levels. Unlimited dimensions are treated @@ -643,7 +762,7 @@ end subroutine set_variable_metadata ! scream decides to allow for other "unlimited" dimensions to be used our ! input/output than this routine will need to be adjusted. subroutine eam_update_time(filename,time) - use pionfput_mod, only: PIO_put_var => put_var + use pio, only: PIO_put_var character(len=*), intent(in) :: filename ! PIO filename real(c_double), intent(in) :: time @@ -660,8 +779,9 @@ subroutine eam_update_time(filename,time) if (time>=0) ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,(/ pio_atm_file%numRecs /), (/ 1 /), (/ time /)) end subroutine eam_update_time !=====================================================================! - ! Assign institutions to header metadata for a specific pio output file. + ! Assign institutions to header metadata for a specific pio output file. subroutine eam_pio_createHeader(File) + use pio, only: PIO_put_att type(file_desc_t), intent(in) :: File ! Pio file Handle integer :: retval @@ -669,7 +789,7 @@ subroutine eam_pio_createHeader(File) ! We are able to have EAMxx directly set most attributes in the HEADER ! except the list of institutions which appears to have a string that is too ! long to accomodate using `set_str_attribute` as it is currently defined. - ! So we keep the setting of institutions here. + ! So we keep the setting of institutions here. ! TODO: revise the set_str_attribute code to allow the ! scream_output_manager.cpp to handle institutions too. ! NOTE: The use of //char(10)// causes each institution to be written on it's own line, makes it easier to read. @@ -693,7 +813,7 @@ subroutine eam_init_pio_subsystem(mpicom,atm_id) shr_pio_getiotype, shr_pio_getioformat #else use pio_types, only: pio_rearr_subset, PIO_iotype_netcdf, PIO_64BIT_DATA - use piolib_mod, only: pio_init + use pio, only: pio_init integer :: ierr, stride, atm_rank, atm_size, num_aggregator #endif @@ -733,7 +853,7 @@ subroutine eam_init_pio_subsystem(mpicom,atm_id) pio_file_list_back => null() pio_file_list_front => null() - ! Init the iodecomp + ! Init the iodecomp iodesc_list_top => null() end subroutine eam_init_pio_subsystem @@ -750,7 +870,7 @@ end function is_eam_pio_subsystem_inited !=====================================================================! ! Create a pio netCDF file with the appropriate name. subroutine eam_pio_createfile(File,fname) - use piolib_mod, only: pio_createfile + use pio, only: pio_createfile use pio_types, only: pio_clobber type(file_desc_t), intent(inout) :: File ! Pio file Handle @@ -767,7 +887,7 @@ end subroutine eam_pio_createfile !=====================================================================! ! Open an already existing netCDF file. subroutine eam_pio_openfile(pio_file,fname) - use piolib_mod, only: pio_openfile + use pio, only: pio_openfile use pio_types, only: pio_write, pio_nowrite type(pio_atm_file_t), pointer, intent(in) :: pio_file ! Pointer to pio file struct associated with this filename @@ -776,7 +896,7 @@ subroutine eam_pio_openfile(pio_file,fname) integer :: retval ! PIO error return value integer :: mode ! Mode for how to handle the new file - if (pio_file%purpose .eq. file_purpose_in) then + if (is_read(pio_file%purpose)) then mode = pio_nowrite else mode = pio_write @@ -784,12 +904,16 @@ subroutine eam_pio_openfile(pio_file,fname) retval = pio_openfile(pio_subsystem,pio_file%pioFileDesc,pio_iotype,fname,mode) call errorHandle("PIO ERROR: unable to open file: "//trim(fname),retval) + if (is_append(pio_file%purpose)) then + pio_file%is_enddef = .true. + endif + end subroutine eam_pio_openfile !=====================================================================! ! Close a netCDF file. To be done as a last step after all input or output ! for that file has been finished. subroutine eam_pio_closefile(fname) - use piolib_mod, only: PIO_syncfile, PIO_closefile + use pio, only: PIO_syncfile, PIO_closefile character(len=*), intent(in) :: fname ! Pio file name !-- @@ -801,9 +925,10 @@ subroutine eam_pio_closefile(fname) ! Find the pointer for this file call lookup_pio_atm_file(trim(fname),pio_atm_file,found,pio_file_list_ptr) + if (found) then if (pio_atm_file%num_customers .eq. 1) then - if (pio_atm_file%purpose .eq. file_purpose_out) then + if ( is_write(pio_atm_file%purpose) ) then call PIO_syncfile(pio_atm_file%pioFileDesc) endif call PIO_closefile(pio_atm_file%pioFileDesc) @@ -815,12 +940,14 @@ subroutine eam_pio_closefile(fname) do while (associated(curr_var_list)) var => curr_var_list%var ! The actual variable pointer if (associated(var)) then - ! Remove this variable as a customer of the associated iodesc - var%iodesc_list%num_customers = var%iodesc_list%num_customers - 1 - ! Dellocate select memory from this variable. Note we can't just - ! deallocate the whole var structure because this would also - ! deallocate the iodesc_list. - call deallocate_hist_var_t(var) + if (associated(var%iodesc_list)) then + ! Remove this variable as a customer of the associated iodesc + var%iodesc_list%num_customers = var%iodesc_list%num_customers - 1 + ! Dellocate select memory from this variable. Note we can't just + ! deallocate the whole var structure because this would also + ! deallocate the iodesc_list. + call deallocate_hist_var_t(var) + end if ! associated(var%iodesc_list) end if ! associated(var) curr_var_list => curr_var_list%next ! Move on to the next variable end do ! associated(curr_var_list) @@ -858,12 +985,35 @@ subroutine eam_pio_closefile(fname) end subroutine eam_pio_closefile !=====================================================================! - ! Helper function to debug list of decomps + ! Flushes IO buffers to file + subroutine eam_pio_flush_file(fname) + use pio, only: PIO_syncfile + + character(len=*), intent(in) :: fname ! Pio file name + !-- + type(pio_atm_file_t),pointer :: pio_atm_file + logical :: found + + ! Find the pointer for this file + call lookup_pio_atm_file(trim(fname),pio_atm_file,found) + + if (found) then + if ( is_write(pio_atm_file%purpose) ) then + call PIO_syncfile(pio_atm_file%pioFileDesc) + else + call errorHandle("PIO ERROR: unable to flush file: "//trim(fname)//", is not open in write mode",-999) + endif + else + call errorHandle("PIO ERROR: unable to flush file: "//trim(fname)//", was not found",-999) + end if + end subroutine eam_pio_flush_file +!=====================================================================! + ! Helper function to debug list of decomps subroutine print_decomp() type(iodesc_list_t), pointer :: iodesc_ptr integer :: total - integer :: cnt + integer :: cnt logical :: assoc if (associated(iodesc_list_top)) then @@ -896,7 +1046,7 @@ end subroutine print_decomp subroutine deallocate_hist_var_t(var) type(hist_var_t), pointer :: var - + deallocate(var%compdof) deallocate(var%dimid) deallocate(var%dimlen) @@ -910,7 +1060,7 @@ end subroutine deallocate_hist_var_t ! trying to keep decomps persistent so they can be reused. Thus, calling ! this routine is optional. subroutine free_decomp() - use piolib_mod, only: PIO_freedecomp + use pio, only: PIO_freedecomp type(iodesc_list_t), pointer :: iodesc_ptr, next ! Free all decompositions from PIO @@ -945,7 +1095,7 @@ end subroutine free_decomp ! Finalize a PIO session within scream. Close all open files and deallocate ! the pio_subsystem session. subroutine eam_pio_finalize() - use piolib_mod, only: PIO_finalize, pio_freedecomp + use pio, only: PIO_finalize, pio_freedecomp ! May not be needed, possibly handled by PIO directly. #if !defined(SCREAM_CIME_BUILD) @@ -997,7 +1147,7 @@ end subroutine finalize_scream_session integer :: ierr if (retVal .ne. PIO_NOERR) then - write(*,'(I8,2x,A200)') retVal,trim(errMsg) + write(*,'(I8,2x,A512)') retVal,trim(errMsg) ! Kill run call eam_pio_finalize() call finalize_scream_session() @@ -1009,7 +1159,7 @@ end subroutine errorHandle ! Determine the unique pio_decomposition for this output grid, if it hasn't ! been defined create a new one. subroutine get_decomp(tag,dtype,dimension_len,compdof,iodesc_list) - use piolib_mod, only: pio_initdecomp + use pio, only: pio_initdecomp ! TODO: CAM code creates the decomp tag for the user. Theoretically it is ! unique because it is based on dimensions and datatype. But the tag ends ! up not being very descriptive. The todo item is to revisit how tags are @@ -1019,7 +1169,7 @@ subroutine get_decomp(tag,dtype,dimension_len,compdof,iodesc_list) integer, intent(in) :: dtype ! Datatype associated with the output integer, intent(in) :: dimension_len(:) ! Array of the dimension lengths for this decomp integer(kind=pio_offset_kind), intent(in) :: compdof(:) ! The degrees of freedom this rank is responsible for - type(iodesc_list_t), pointer :: iodesc_list ! The pio decomposition list that holds this iodesc + type(iodesc_list_t), pointer :: iodesc_list ! The pio decomposition list that holds this iodesc logical :: found ! Whether a decomp has been found among the previously defined decompositions type(iodesc_list_t),pointer :: curr, prev ! Used to toggle through the recursive list of decompositions @@ -1068,6 +1218,7 @@ subroutine get_decomp(tag,dtype,dimension_len,compdof,iodesc_list) call pio_initdecomp(pio_subsystem, dtype, dimension_len, compdof, curr%iodesc, rearr=pio_rearranger) curr%iodesc_set = .true. end if + curr%num_customers = 0 end if iodesc_list => curr @@ -1135,18 +1286,23 @@ subroutine set_decomp(filename) curr => current_atm_file%var_list_top do while (associated(curr)) + ! Skip already deallocated vars, and vars for which decomp was already set + ! NOTE: fortran does not mandate/prohibit logical op short circuit, so do the + ! two following if statements separately if (associated(curr%var)) then - hist_var => curr%var - if (.not.associated(hist_var)) call errorHandle("PIO ERROR: unable to set decomp for file, var: "//trim(current_atm_file%filename)//", "//trim(hist_var%name)//". Set DOF.",999) - ! Assign decomp - if (hist_var%has_t_dim) then - loc_len = max(1,hist_var%numdims-1) - call get_decomp(hist_var%pio_decomp_tag,hist_var%dtype,hist_var%dimlen(:loc_len),hist_var%compdof,hist_var%iodesc_list) - else - call get_decomp(hist_var%pio_decomp_tag,hist_var%dtype,hist_var%dimlen,hist_var%compdof,hist_var%iodesc_list) - end if - hist_var%iodesc => hist_var%iodesc_list%iodesc - hist_var%iodesc_list%num_customers = hist_var%iodesc_list%num_customers + 1 ! Add this variable as a customer of this pio decomposition + if (.not. associated(curr%var%iodesc)) then + hist_var => curr%var + if (.not.associated(hist_var)) call errorHandle("PIO ERROR: unable to set decomp for file, var: "//trim(current_atm_file%filename)//", "//trim(hist_var%name)//". Set DOF.",999) + ! Assign decomp + if (hist_var%has_t_dim) then + loc_len = max(1,hist_var%numdims-1) + call get_decomp(hist_var%pio_decomp_tag,hist_var%dtype,hist_var%dimlen(:loc_len),hist_var%compdof,hist_var%iodesc_list) + else + call get_decomp(hist_var%pio_decomp_tag,hist_var%dtype,hist_var%dimlen,hist_var%compdof,hist_var%iodesc_list) + end if + hist_var%iodesc => hist_var%iodesc_list%iodesc + hist_var%iodesc_list%num_customers = hist_var%iodesc_list%num_customers + 1 ! Add this variable as a customer of this pio decomposition + endif end if curr => curr%next end do @@ -1206,6 +1362,7 @@ end subroutine lookup_pio_atm_file !=====================================================================! ! Create a new pio file pointer based on filename. subroutine get_pio_atm_file(filename,pio_file,purpose) + use pio, only: PIO_inq_dimid, PIO_inq_dimlen character(len=*),intent(in) :: filename ! Name of file to be found type(pio_atm_file_t), pointer :: pio_file ! Pointer to pio_atm_output structure associated with this filename @@ -1216,23 +1373,25 @@ subroutine get_pio_atm_file(filename,pio_file,purpose) integer :: ierr, time_id - ! Sanity check - if (purpose .ne. file_purpose_in .and. purpose .ne. file_purpose_out) then - call errorHandle("PIO Error: unrecognized file purpose for file '"//filename//"'.",-999) + ! Sanity checks + if ( .not. (is_read(purpose) .or. is_write(purpose) .or. is_append(purpose)) ) then + call errorHandle("PIO Error: unrecognized open mode requested for file '"//filename//"'.",-999) + endif + if ( is_read(purpose) .and. is_write(purpose) ) then + call errorHandle("PIO Error: both READ and WRITE mode requested for file '"//filename//"'.",-999) + endif + if ( is_read(purpose) .and. is_append(purpose) ) then + call errorHandle("PIO Error: APPEND mode requested along with READ mode for file '"//filename//"'.",-999) endif ! If the file already exists, return that file call lookup_pio_atm_file(trim(filename),pio_file,found) if (found) then - if (purpose .ne. file_purpose_in .or. & - pio_file%purpose .ne. file_purpose_in ) then + if (is_write(purpose) .or. is_write(pio_file%purpose) ) then ! We only allow multiple customers of the file if they all use it in read mode. call errorHandle("PIO Error: file '"//trim(filename)//"' was already open for writing.",-999) - else - pio_file%purpose = purpose - call eam_pio_openfile(pio_file,trim(pio_file%filename)) - pio_file%num_customers = pio_file%num_customers + 1 endif + pio_file%num_customers = pio_file%num_customers + 1 else allocate(new_list_item) allocate(new_list_item%pio_file) @@ -1243,12 +1402,9 @@ subroutine get_pio_atm_file(filename,pio_file,purpose) pio_file%numRecs = 0 pio_file%num_customers = 1 pio_file%purpose = purpose - if (purpose == file_purpose_out) then ! Will be used for output. Set numrecs to zero and create the new file. - call eam_pio_createfile(pio_file%pioFileDesc,trim(pio_file%filename)) - call eam_pio_createHeader(pio_file%pioFileDesc) - elseif (purpose == file_purpose_in) then ! Will be used for input, just open it + if (is_read(purpose) .or. is_append(purpose)) then + ! Either read or append to existing file. Either way, file must exist on disk call eam_pio_openfile(pio_file,trim(pio_file%filename)) - pio_file%is_enddef = .true. ! Files open in read mode are in data mode already ! Update the numRecs to match the number of recs in this file. ierr = pio_inq_dimid(pio_file%pioFileDesc,"time",time_id) if (ierr.ne.0) then @@ -1259,6 +1415,10 @@ subroutine get_pio_atm_file(filename,pio_file,purpose) ierr = pio_inq_dimlen(pio_file%pioFileDesc,time_id,pio_file%numRecs) call errorHandle("EAM_PIO ERROR: Unable to determine length for dimension time in file "//trim(pio_file%filename),ierr) end if + elseif (is_write(purpose)) then + ! New output file + call eam_pio_createfile(pio_file%pioFileDesc,trim(pio_file%filename)) + call eam_pio_createHeader(pio_file%pioFileDesc) else call errorHandle("PIO Error: get_pio_atm_file with filename = "//trim(filename)//", purpose (int) assigned to this lookup is not valid" ,-999) end if @@ -1283,14 +1443,13 @@ end subroutine get_pio_atm_file ! If the input arg time_index is not provided, then it is assumed the user wants ! the last time entry. If time_index is present, it MUST be valid function read_time_at_index(filename,time_index) result(val) - use pio, only: PIO_get_var - use pio_nf, only: PIO_inq_varid + use pio, only: PIO_get_var, PIO_inq_varid, PIO_inq_dimid, PIO_inq_dimlen character(len=*), intent(in) :: filename integer, intent(in), optional :: time_index real(c_double) :: val real(c_double) :: val_buf(1) - + type(pio_atm_file_t), pointer :: pio_atm_file logical :: found integer :: dim_id, time_len, ierr @@ -1340,10 +1499,8 @@ end function read_time_at_index ! !--------------------------------------------------------------------------- subroutine grid_write_darray_float(filename, varname, buf, buf_size) - use pionfput_mod, only: PIO_put_var => put_var - use piolib_mod, only: PIO_setframe + use pio, only: PIO_put_var, PIO_setframe, PIO_write_darray use pio_types, only: PIO_max_var_dims - use piodarray, only: PIO_write_darray ! Dummy arguments character(len=*), intent(in) :: filename ! PIO filename @@ -1389,10 +1546,8 @@ subroutine grid_write_darray_float(filename, varname, buf, buf_size) call errorHandle( 'eam_grid_write_darray_float: Error writing variable '//trim(varname),ierr) end subroutine grid_write_darray_float subroutine grid_write_darray_double(filename, varname, buf, buf_size) - use pionfput_mod, only: PIO_put_var => put_var + use pio, only: PIO_put_var, PIO_setframe, PIO_write_darray use pio_types, only: PIO_max_var_dims - use piolib_mod, only: PIO_setframe - use piodarray, only: PIO_write_darray ! Dummy arguments character(len=*), intent(in) :: filename ! PIO filename @@ -1438,10 +1593,8 @@ subroutine grid_write_darray_double(filename, varname, buf, buf_size) call errorHandle( 'eam_grid_write_darray_double: Error writing variable '//trim(varname),ierr) end subroutine grid_write_darray_double subroutine grid_write_darray_int(filename, varname, buf, buf_size) - use pionfput_mod, only: PIO_put_var => put_var - use piolib_mod, only: PIO_setframe + use pio, only: PIO_put_var, PIO_setframe, PIO_write_darray use pio_types, only: PIO_max_var_dims - use piodarray, only: PIO_write_darray ! Dummy arguments character(len=*), intent(in) :: filename ! PIO filename @@ -1504,8 +1657,7 @@ end subroutine grid_write_darray_int ! !--------------------------------------------------------------------------- subroutine grid_read_darray_double(filename, varname, buf, buf_size, time_index) - use piolib_mod, only: PIO_setframe - use piodarray, only: PIO_read_darray + use pio, only: PIO_setframe, PIO_read_darray ! Dummy arguments character(len=*), intent(in) :: filename ! PIO filename @@ -1531,7 +1683,7 @@ subroutine grid_read_darray_double(filename, varname, buf, buf_size, time_index) ! Otherwise default to the last time_index in the file call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(pio_atm_file%numRecs,kind=pio_offset_kind)) end if - + ! We don't want the extent along the 'time' dimension var_size = SIZE(var%compdof) @@ -1540,8 +1692,7 @@ subroutine grid_read_darray_double(filename, varname, buf, buf_size, time_index) call errorHandle( 'eam_grid_read_darray_double: Error reading variable '//trim(varname),ierr) end subroutine grid_read_darray_double subroutine grid_read_darray_float(filename, varname, buf, buf_size, time_index) - use piolib_mod, only: PIO_setframe - use piodarray, only: PIO_read_darray + use pio, only: PIO_setframe, PIO_read_darray ! Dummy arguments character(len=*), intent(in) :: filename ! PIO filename @@ -1567,7 +1718,7 @@ subroutine grid_read_darray_float(filename, varname, buf, buf_size, time_index) ! Otherwise default to the last time_index in the file call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(pio_atm_file%numRecs,kind=pio_offset_kind)) end if - + ! We don't want the extent along the 'time' dimension var_size = SIZE(var%compdof) @@ -1576,8 +1727,7 @@ subroutine grid_read_darray_float(filename, varname, buf, buf_size, time_index) call errorHandle( 'eam_grid_read_darray_float: Error reading variable '//trim(varname),ierr) end subroutine grid_read_darray_float subroutine grid_read_darray_int(filename, varname, buf, buf_size, time_index) - use piolib_mod, only: PIO_setframe - use piodarray, only: PIO_read_darray + use pio, only: PIO_setframe, PIO_read_darray ! Dummy arguments character(len=*), intent(in) :: filename ! PIO filename @@ -1603,7 +1753,7 @@ subroutine grid_read_darray_int(filename, varname, buf, buf_size, time_index) ! Otherwise default to the last time_index in the file call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(pio_atm_file%numRecs,kind=pio_offset_kind)) end if - + ! We don't want the extent along the 'time' dimension var_size = SIZE(var%compdof) @@ -1643,12 +1793,12 @@ subroutine convert_int_2_str(int_in,str_out) elseif (abs(int_in)<1e9) then fmt_str = trim(fmt_str)//"I9)" endif - + if (int_in < 0) then write(str_out,fmt_str) "n", int_in else write(str_out,fmt_str) int_in - end if + end if end subroutine convert_int_2_str @@ -1686,5 +1836,21 @@ subroutine get_coord (filename,shortname,hist_coord) hist_coord => curr%coord end subroutine get_coord + + function is_read (purpose) + integer, intent(in) :: purpose + logical :: is_read + is_read = iand(purpose,file_purpose_in) .ne. 0 + end function is_read + function is_write (purpose) + integer, intent(in) :: purpose + logical :: is_write + is_write = iand(purpose,file_purpose_out) .ne. 0 + end function is_write + function is_append (purpose) + integer, intent(in) :: purpose + logical :: is_append + is_append = iand(purpose,file_purpose_app) .ne. 0 + end function is_append !=====================================================================! end module scream_scorpio_interface diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 7b9cc770c14b..676b72f60aee 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -16,6 +16,7 @@ extern "C" { // Fortran routines to be called from C++ void register_file_c2f(const char*&& filename, const int& mode); + int get_file_mode_c2f(const char*&& filename); void set_decomp_c2f(const char*&& filename); void set_dof_c2f(const char*&& filename,const char*&& varname,const Int dof_len,const std::int64_t *x_dof); void grid_read_data_array_c2f_int(const char*&& filename, const char*&& varname, const Int time_index, int *buf, const int buf_size); @@ -28,15 +29,19 @@ extern "C" { void eam_init_pio_subsystem_c2f(const int mpicom, const int atm_id); void eam_pio_finalize_c2f(); void eam_pio_closefile_c2f(const char*&& filename); + void eam_pio_flush_file_c2f(const char*&& filename); void pio_update_time_c2f(const char*&& filename,const double time); void register_dimension_c2f(const char*&& filename, const char*&& shortname, const char*&& longname, const int global_length, const bool partitioned); void register_variable_c2f(const char*&& filename, const char*&& shortname, const char*&& longname, const char*&& units, const int numdims, const char** var_dimensions, const int dtype, const int nc_dtype, const char*&& pio_decomp_tag); - void set_variable_metadata_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, const char*&& meta_val); - void get_variable_c2f(const char*&& filename,const char*&& shortname, const char*&& longname, - const int numdims, const char** var_dimensions, - const int dtype, const char*&& pio_decomp_tag); + void set_variable_metadata_char_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, const char*&& meta_val); + void set_variable_metadata_float_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, const float meta_val); + void set_variable_metadata_double_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, const double meta_val); + float get_variable_metadata_float_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name); + double get_variable_metadata_double_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name); + void get_variable_metadata_char_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, char*&& meta_val); + void eam_pio_redef_c2f(const char*&& filename); void eam_pio_enddef_c2f(const char*&& filename); bool is_enddef_c2f(const char*&& filename); } // extern C @@ -48,6 +53,8 @@ namespace scorpio { int nctype (const std::string& type) { if (type=="int") { return PIO_INT; + } else if (type=="int64") { + return PIO_INT64; } else if (type=="float" || type=="single") { return PIO_FLOAT; } else if (type=="double") { @@ -62,6 +69,12 @@ int nctype (const std::string& type) { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported data type '" + type + "'.\n"); } } +std::string nctype2str (const int type) { + for (auto t : {"int", "int64", "float", "double"}) { + if (nctype(t)==type) return t; + } + return "UNKNOWN"; +} /* ----------------------------------------------------------------- */ void eam_init_pio_subsystem(const ekat::Comm& comm) { MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); @@ -89,6 +102,9 @@ void eam_pio_closefile(const std::string& filename) { eam_pio_closefile_c2f(filename.c_str()); } +void eam_flush_file(const std::string& filename) { + eam_pio_flush_file_c2f(filename.c_str()); +} /* ----------------------------------------------------------------- */ void set_decomp(const std::string& filename) { @@ -132,6 +148,33 @@ int get_dimlen(const std::string& filename, const std::string& dimname) return len; } /* ----------------------------------------------------------------- */ +bool has_dim (const std::string& filename, const std::string& dimname) +{ + int ncid, dimid, err; + + bool was_open = is_file_open_c2f(filename.c_str(),-1); + if (not was_open) { + register_file(filename,Read); + } + + ncid = get_file_ncid_c2f (filename.c_str()); + err = PIOc_inq_dimid(ncid,dimname.c_str(),&dimid); + if (err==PIO_EBADDIM) { + return false; + } + + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while retrieving dimension id.\n" + " - filename : " + filename + "\n" + " - dimname : " + dimname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + if (not was_open) { + eam_pio_closefile(filename); + } + + return true; +} +/* ----------------------------------------------------------------- */ bool has_variable (const std::string& filename, const std::string& varname) { int ncid, varid, err; @@ -168,31 +211,210 @@ void pio_update_time(const std::string& filename, const double time) { pio_update_time_c2f(filename.c_str(),time); } /* ----------------------------------------------------------------- */ -void register_dimension(const std::string &filename, const std::string& shortname, const std::string& longname, const int length, const bool partitioned) { +void register_dimension(const std::string &filename, const std::string& shortname, const std::string& longname, const int length, const bool partitioned) +{ + int mode = get_file_mode_c2f(filename.c_str()); + std::string mode_str = mode==Read ? "Read" : (mode==Write ? "Write" : "Append"); + if (mode!=Write) { + // Ensure the dimension already exists, and that it has the correct size (if not unlimited) + int ncid,dimid,unlimid,err; + + ncid = get_file_ncid_c2f (filename.c_str()); + err = PIOc_inq_dimid(ncid,shortname.c_str(),&dimid); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Could not retrieve dimension id from file open in " + mode_str + " mode.\n" + " - filename: " + filename + "\n" + " - dimension : " + shortname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + + err = PIOc_inq_unlimdim(ncid,&unlimid); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong querying for the unlimited dimension id.\n" + " - filename: " + filename + "\n" + " - pio error: " + std::to_string(err) + "\n"); + if (length==0) { + EKAT_REQUIRE_MSG ( unlimid==dimid, + "Error! Input dimension is unlimited, but does not appear to be unlimited in the file (open in " + mode_str + " mode).\n" + " - filename: " + filename + "\n" + " - dimension : " + shortname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + } else { + EKAT_REQUIRE_MSG ( unlimid!=dimid, + "Error! Input dimension is not unlimited, but it appears to be unlimited in the file (open in " + mode_str + " mode).\n" + " - filename: " + filename + "\n" + " - dimension : " + shortname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + + int len_from_file = get_dimlen(filename,shortname); + EKAT_REQUIRE_MSG (length==len_from_file, + "Error! Input dimension length does not match the one from the file (open in " + mode_str + " mode).\n" + " - filename: " + filename + "\n" + " - dimension : " + shortname + "\n" + " - input dim length: " + std::to_string(length) + "\n" + " - file dim length : " + std::to_string(len_from_file) + "\n"); + } + } register_dimension_c2f(filename.c_str(), shortname.c_str(), longname.c_str(), length, partitioned); } /* ----------------------------------------------------------------- */ -void get_variable(const std::string &filename, const std::string& shortname, const std::string& longname, - const std::vector& var_dimensions, - const std::string& dtype, const std::string& pio_decomp_tag) { - - /* Convert the vector of strings that contains the variable dimensions to a char array */ - const int numdims = var_dimensions.size(); - std::vector var_dimensions_c(numdims); - for (int ii = 0;ii& var_dimensions, + const std::string& dtype, const std::string& pio_decomp_tag) +{ + // This overload does not require to specify an nc data type, so it *MUST* be used when the + // file access mode is either Read or Append. Either way, a) the var should be on file already, + // and b) so should be the dimensions + EKAT_REQUIRE_MSG (has_variable(filename,shortname), + "Error! This overload of register_variable *assumes* the variable is already in the file, but wasn't found.\n" + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n"); + for (const auto& dimname : var_dimensions) { + int len = get_dimlen (filename,dimname); + // WARNING! If the dimension was not yet registered, it will be registered as a NOT partitioned dim. + // If this dim should be partitioned, then register it *before* the variable + register_dimension(filename,dimname,dimname,len,false); } - get_variable_c2f(filename.c_str(), shortname.c_str(), longname.c_str(), - numdims, var_dimensions_c.data(), nctype(dtype), pio_decomp_tag.c_str()); + register_variable(filename,shortname,longname,"",var_dimensions,dtype,"",pio_decomp_tag); } -/* ----------------------------------------------------------------- */ void register_variable(const std::string &filename, const std::string& shortname, const std::string& longname, - const std::string& units, const std::vector& var_dimensions, - const std::string& dtype, const std::string& nc_dtype, const std::string& pio_decomp_tag) { + const std::string& units_in, const std::vector& var_dimensions, + const std::string& dtype, const std::string& nc_dtype_in, const std::string& pio_decomp_tag) +{ + // Local copies, since we can modify them in case of defaults + auto units = units_in; + auto nc_dtype = nc_dtype_in; + + int mode = get_file_mode_c2f(filename.c_str()); + std::string mode_str = mode==Read ? "Read" : (mode==Write ? "Write" : "Append"); + + bool has_var = has_variable(filename,shortname); + if (mode==Write) { + EKAT_REQUIRE_MSG ( units!="" and nc_dtype!="", + "Error! Missing valid units and/or nc_dtype arguments for file open in Write mode.\n" + " - filename: " + filename + "\n"); + } else { + EKAT_REQUIRE_MSG ( has_var, + "Error! Variable not found in file open in " + mode_str + " mode.\n" + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n"); + } + - /* Convert the vector of strings that contains the variable dimensions to a char array */ + if (has_var) { + // The file already exists or the var was already registered. + // Make sure we're registering the var with the same specs + int ncid = get_file_ncid_c2f (filename.c_str()); + int vid,ndims,err; + err = PIOc_inq_varid(ncid,shortname.c_str(),&vid); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong retrieving variable id.\n" + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + err = PIOc_inq_varndims(ncid,vid,&ndims); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong inquiring the number of dimensions of a variable.\n" + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + std::vector dims(ndims); + err = PIOc_inq_vardimid(ncid,vid,dims.data()); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong inquiring the dimensions ids of a variable.\n" + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + std::vector dims_from_file(ndims); + for (int i=0; i0) + if (var_dimensions.size()>0) { + if (mode==Read && (dims_from_file[0]=="time" && var_dimensions[0]!="time")) { + // For Read operations, we may not consider "time" as a field dimension, so if the + // input file has "time", simply disregard it in this check. + dims_from_file.erase(dims_from_file.begin()); + } + } else { + if (mode==Read && (dims_from_file[0]=="time")) { + // For Read operations, we may not consider "time" as a field dimension, so if the + // input file has "time", simply disregard it in this check. + dims_from_file.erase(dims_from_file.begin()); + } + } + std::reverse(dims_from_file.begin(),dims_from_file.end()); + EKAT_REQUIRE_MSG(var_dimensions==dims_from_file, + "Error! Input variable dimensions do not match the ones from the file.\n" + " - filename : " + filename + "\n" + " - input dims: (" + ekat::join(var_dimensions,",") + ")\n" + " - file dims : (" + ekat::join(dims_from_file,",") + ")\n"); + + // Check nc dtype only if user bothered specifying it (if mode=Read, probably the user doesn't care) + nc_type type; + err = PIOc_inq_vartype(ncid,vid,&type); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while inquiring variable data type.\n" + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + EKAT_REQUIRE_MSG (nc_dtype=="" or type==nctype(nc_dtype), + "Error! Input NC data type does not match the one from the file.\n" + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n" + " - input dtype : " + nc_dtype + "\n" + " - file dtype : " + nctype2str(type) + "\n"); + nc_dtype = nctype2str(type); + + // Get var units (if set) + int natts; + err = PIOc_inq_varnatts(ncid,vid,&natts); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while inquiring a variable's number of attributes.\n" + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + std::string units_from_file(PIO_MAX_NAME,'\0'); + std::string att_name(PIO_MAX_NAME,'\0'); + for (int i=0; i var_dimensions_c(numdims); for (int ii = 0;ii=0, "[get_any_attribute] Error! Could not retrieve file ncid.\n" " - filename : " + filename + "\n"); - int varid = PIO_GLOBAL; + int varid; int err; + if (var_name=="GLOBAL") { + varid = PIO_GLOBAL; + } else { + err = PIOc_inq_varid(ncid, var_name.c_str(), &varid); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "[get_any_attribute] Error! Something went wrong while inquiring variable id.\n" + " - filename : " + filename + "\n" + " - variable : " + var_name + "\n" + " - attribute: " + att_name + "\n" + " - pio error: " << err << "\n"); + } nc_type type; PIO_Offset len; @@ -224,12 +488,14 @@ ekat::any get_any_attribute (const std::string& filename, const std::string& att EKAT_REQUIRE_MSG (err==PIO_NOERR, "[get_any_attribute] Error! Something went wrong while inquiring global attribute.\n" " - filename : " + filename + "\n" + " - variable : " + var_name + "\n" " - attribute: " + att_name + "\n" " - pio error: " << err << "\n"); EKAT_REQUIRE_MSG (len==1 || type==PIO_CHAR, "[get_any_attribute] Error! Only single value attributes allowed.\n" " - filename : " + filename + "\n" + " - variable : " + var_name + "\n" " - attribute: " + att_name + "\n" " - nc type : " << type << "\n" " - att len : " << len << "\n"); @@ -254,12 +520,14 @@ ekat::any get_any_attribute (const std::string& filename, const std::string& att } else { EKAT_ERROR_MSG ("[get_any_attribute] Error! Unsupported/unrecognized nc type.\n" " - filename : " + filename + "\n" + " - variable : " + var_name + "\n" " - attribute: " + att_name + "\n" " - nc type : " << type << "\n"); } EKAT_REQUIRE_MSG (err==PIO_NOERR, "[get_any_attribute] Error! Something went wrong while inquiring global attribute.\n" " - filename : " + filename + "\n" + " - variable : " + var_name + "\n" " - attribute: " + att_name + "\n" " - pio error: " << err << "\n"); @@ -325,6 +593,10 @@ void eam_pio_enddef(const std::string &filename) { eam_pio_enddef_c2f(filename.c_str()); } /* ----------------------------------------------------------------- */ +void eam_pio_redef(const std::string &filename) { + eam_pio_redef_c2f(filename.c_str()); +} +/* ----------------------------------------------------------------- */ template<> void grid_read_data_array(const std::string &filename, const std::string &varname, const int time_index, int *hbuf, const int buf_size) { diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 4f1c3a4dd1a7..4289265d07aa 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -21,7 +21,8 @@ namespace scorpio { // in the scream_scorpio_interface F90 module enum FileMode { Read = 1, - Write = 2 + Append = 2, + Write = 4 }; /* All scorpio usage requires that the pio_subsystem is initialized. Happens only once per simulation */ void eam_init_pio_subsystem(const ekat::Comm& comm); @@ -30,10 +31,12 @@ namespace scorpio { void eam_pio_finalize(); /* Close a file currently open in scorpio */ void eam_pio_closefile(const std::string& filename); + void eam_flush_file(const std::string& filename); /* Register a new file to be used for input/output with the scorpio module */ void register_file(const std::string& filename, const FileMode mode); /* Sets the IO decompostion for all variables in a particular filename. Required after all variables have been registered. Called once per file. */ int get_dimlen(const std::string& filename, const std::string& dimname); + bool has_dim(const std::string& filename, const std::string& dimname); bool has_variable (const std::string& filename, const std::string& varname); void set_decomp(const std::string& filename); /* Sets the degrees-of-freedom for a particular variable in a particular file. Called once for each variable, for each file. */ @@ -44,16 +47,23 @@ namespace scorpio { void register_variable(const std::string& filename, const std::string& shortname, const std::string& longname, const std::string& units, const std::vector& var_dimensions, const std::string& dtype, const std::string& nc_dtype, const std::string& pio_decomp_tag); + void register_variable(const std::string& filename, const std::string& shortname, const std::string& longname, + const std::vector& var_dimensions, + const std::string& dtype, const std::string& pio_decomp_tag); void set_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, const std::string& meta_val); + void set_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, const float meta_val); + void set_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, const double meta_val); + void get_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, float& meta_val); + void get_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, double& meta_val); + void get_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, std::string& meta_val); /* Register a variable with a file. Called during the file setup, for an input stream. */ - void get_variable(const std::string& filename,const std::string& shortname, const std::string& longname, - const std::vector& var_dimensions, - const std::string& dtype, const std::string& pio_decomp_tag); ekat::any get_any_attribute (const std::string& filename, const std::string& att_name); + ekat::any get_any_attribute (const std::string& filename, const std::string& var_name, const std::string& att_name); void set_any_attribute (const std::string& filename, const std::string& att_name, const ekat::any& att); /* End the definition phase for a scorpio file. Last thing called after all dimensions, variables, dof's and decomps have been set. Called once per file. * Mandatory before writing or reading can happend on file. */ void eam_pio_enddef(const std::string &filename); + void eam_pio_redef(const std::string &filename); /* Called each timestep to update the timesnap for the last written output. */ void pio_update_time(const std::string &filename, const double time); @@ -80,7 +90,6 @@ namespace scorpio { { ekat::any a(att); set_any_attribute(filename,att_name,a); - } // Shortcut to write/read to/from YYYYMMDD/HHMMSS attributes in the NC file @@ -95,21 +104,15 @@ extern "C" { // If mode<0, then simply checks if file is open, regardless of mode bool is_file_open_c2f(const char*&& filename, const int& mode); /* Query a netCDF file for the time variable */ + bool is_enddef_c2f(const char*&& filename); double read_time_at_index_c2f(const char*&& filename, const int& time_index); double read_curr_time_c2f(const char*&& filename); + /* Query a netCDF file for the metadata associated w/ a variable */ + float get_variable_metadata_float_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name); + double get_variable_metadata_double_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name); } // extern "C" -// The strings returned by e2str(const FieldTag&) are different from -// what existing nc files are already using. Besides upper/lower case -// differences, the column dimension (COL) is 'ncol' in nc files, -// but we'd like to keep 'COL' when printing our layouts, so we -// create this other mini helper function to get the name of a tag -// that is compatible with nc files. Note that tags that make no -// sense for an nc file are omitted. Namely, all those that have a -// field-dependent extent, such as vector dimensions. Those have to -// be "unpacked", storing a separate variable for each slice. - } // namespace scorpio } // namespace scream -#endif // define SCREAM_SCORPIO_INTERFACE_HPP +#endif // define SCREAM_SCORPIO_INTERFACE_HPP diff --git a/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 b/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 index a34665389da9..9e50842b245e 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 @@ -38,6 +38,24 @@ function get_file_ncid_c2f(filename_in) result(ncid) bind(c) endif end function get_file_ncid_c2f !=====================================================================! + function get_file_mode_c2f(filename_in) result(mode) bind(c) + use scream_scorpio_interface, only : lookup_pio_atm_file, pio_atm_file_t + + type(c_ptr), intent(in) :: filename_in + + type(pio_atm_file_t), pointer :: atm_file + character(len=256) :: filename + integer(kind=c_int) :: mode + logical :: found + + call convert_c_string(filename_in,filename) + call lookup_pio_atm_file(filename,atm_file,found) + if (found) then + mode = atm_file%purpose + else + mode = 0 + endif + end function get_file_mode_c2f function is_file_open_c2f(filename_in,purpose) result(res) bind(c) use scream_scorpio_interface, only : lookup_pio_atm_file, pio_atm_file_t @@ -52,7 +70,7 @@ function is_file_open_c2f(filename_in,purpose) result(res) bind(c) call convert_c_string(filename_in,filename) call lookup_pio_atm_file(filename,atm_file,found) if (found) then - res = LOGICAL(purpose .eq. 0 .or. atm_file%purpose .eq. purpose,kind=c_bool) + res = LOGICAL(purpose .lt. 0 .or. atm_file%purpose .eq. purpose,kind=c_bool) else res = .false. endif @@ -90,16 +108,18 @@ subroutine set_dof_c2f(filename_in,varname_in,dof_len,dof_vec) bind(c) character(len=256) :: filename character(len=256) :: varname - integer(kind=pio_offset_kind) :: dof_vec_f90(dof_len) integer :: ii + integer(kind=pio_offset_kind), allocatable :: dof_vec_f90(:) call convert_c_string(filename_in,filename) call convert_c_string(varname_in,varname) ! Need to add 1 to the dof_vec because C++ starts indices at 0 not 1: + allocate(dof_vec_f90(dof_len)) do ii = 1,dof_len dof_vec_f90(ii) = dof_vec(ii) + 1 end do call set_dof(trim(filename),trim(varname),dof_len,dof_vec_f90) + deallocate(dof_vec_f90) end subroutine set_dof_c2f !=====================================================================! subroutine eam_pio_closefile_c2f(filename_in) bind(c) @@ -111,6 +131,16 @@ subroutine eam_pio_closefile_c2f(filename_in) bind(c) call eam_pio_closefile(trim(filename)) end subroutine eam_pio_closefile_c2f +!=====================================================================! + subroutine eam_pio_flush_file_c2f(filename_in) bind(c) + use scream_scorpio_interface, only : eam_pio_flush_file + type(c_ptr), intent(in) :: filename_in + character(len=256) :: filename + + call convert_c_string(filename_in,filename) + call eam_pio_flush_file(trim(filename)) + + end subroutine eam_pio_flush_file_c2f !=====================================================================! subroutine pio_update_time_c2f(filename_in,time) bind(c) use scream_scorpio_interface, only : eam_update_time @@ -123,35 +153,6 @@ subroutine pio_update_time_c2f(filename_in,time) bind(c) call eam_update_time(trim(filename),time) end subroutine pio_update_time_c2f -!=====================================================================! - subroutine get_variable_c2f(filename_in, shortname_in, longname_in, numdims, var_dimensions_in, dtype, pio_decomp_tag_in) bind(c) - use scream_scorpio_interface, only : get_variable - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: shortname_in - type(c_ptr), intent(in) :: longname_in - integer(kind=c_int), value, intent(in) :: numdims - type(c_ptr), intent(in) :: var_dimensions_in(numdims) - integer(kind=c_int), value, intent(in) :: dtype - type(c_ptr), intent(in) :: pio_decomp_tag_in - - character(len=256) :: filename - character(len=256) :: shortname - character(len=256) :: longname - character(len=256) :: var_dimensions(numdims) - character(len=256) :: pio_decomp_tag - integer :: ii - - call convert_c_string(filename_in,filename) - call convert_c_string(shortname_in,shortname) - call convert_c_string(longname_in,longname) - call convert_c_string(pio_decomp_tag_in,pio_decomp_tag) - do ii = 1,numdims - call convert_c_string(var_dimensions_in(ii), var_dimensions(ii)) - end do - - call get_variable(filename,shortname,longname,numdims,var_dimensions,dtype,pio_decomp_tag) - - end subroutine get_variable_c2f !=====================================================================! subroutine register_variable_c2f(filename_in, shortname_in, longname_in, & units_in, numdims, var_dimensions_in, & @@ -187,8 +188,8 @@ subroutine register_variable_c2f(filename_in, shortname_in, longname_in, & end subroutine register_variable_c2f !=====================================================================! - subroutine set_variable_metadata_c2f(filename_in, varname_in, metaname_in, metaval_in) bind(c) - use scream_scorpio_interface, only : set_variable_metadata + subroutine set_variable_metadata_char_c2f(filename_in, varname_in, metaname_in, metaval_in) bind(c) + use scream_scorpio_interface, only : set_variable_metadata_char type(c_ptr), intent(in) :: filename_in type(c_ptr), intent(in) :: varname_in type(c_ptr), intent(in) :: metaname_in @@ -204,9 +205,118 @@ subroutine set_variable_metadata_c2f(filename_in, varname_in, metaname_in, metav call convert_c_string(metaname_in,metaname) call convert_c_string(metaval_in,metaval) - call set_variable_metadata(filename,varname,metaname,metaval) + call set_variable_metadata_char(filename,varname,metaname,metaval) + + end subroutine set_variable_metadata_char_c2f +!=====================================================================! + subroutine set_variable_metadata_float_c2f(filename_in, varname_in, metaname_in, metaval_in) bind(c) + use scream_scorpio_interface, only : set_variable_metadata_float + type(c_ptr), intent(in) :: filename_in + type(c_ptr), intent(in) :: varname_in + type(c_ptr), intent(in) :: metaname_in + real(kind=c_float), value, intent(in) :: metaval_in + + character(len=256) :: filename + character(len=256) :: varname + character(len=256) :: metaname + + call convert_c_string(filename_in,filename) + call convert_c_string(varname_in,varname) + call convert_c_string(metaname_in,metaname) + + call set_variable_metadata_float(filename,varname,metaname,metaval_in) + + end subroutine set_variable_metadata_float_c2f +!=====================================================================! + subroutine set_variable_metadata_double_c2f(filename_in, varname_in, metaname_in, metaval_in) bind(c) + use scream_scorpio_interface, only : set_variable_metadata_double + type(c_ptr), intent(in) :: filename_in + type(c_ptr), intent(in) :: varname_in + type(c_ptr), intent(in) :: metaname_in + real(kind=c_double), value, intent(in) :: metaval_in + + character(len=256) :: filename + character(len=256) :: varname + character(len=256) :: metaname + + call convert_c_string(filename_in,filename) + call convert_c_string(varname_in,varname) + call convert_c_string(metaname_in,metaname) + + call set_variable_metadata_double(filename,varname,metaname,metaval_in) - end subroutine set_variable_metadata_c2f + end subroutine set_variable_metadata_double_c2f +!=====================================================================! + function get_variable_metadata_float_c2f(filename_in, varname_in, metaname_in) result(metaval_out) bind(c) + use scream_scorpio_interface, only : get_variable_metadata_float + type(c_ptr), intent(in) :: filename_in + type(c_ptr), intent(in) :: varname_in + type(c_ptr), intent(in) :: metaname_in + real(kind=c_float) :: metaval_out + + character(len=256) :: filename + character(len=256) :: varname + character(len=256) :: metaname + + call convert_c_string(filename_in,filename) + call convert_c_string(varname_in,varname) + call convert_c_string(metaname_in,metaname) + + metaval_out = get_variable_metadata_float(filename,varname,metaname) + + end function get_variable_metadata_float_c2f +!=====================================================================! + function get_variable_metadata_double_c2f(filename_in, varname_in, metaname_in) result(metaval_out) bind(c) + use scream_scorpio_interface, only : get_variable_metadata_double + type(c_ptr), intent(in) :: filename_in + type(c_ptr), intent(in) :: varname_in + type(c_ptr), intent(in) :: metaname_in + real(kind=c_double) :: metaval_out + + character(len=256) :: filename + character(len=256) :: varname + character(len=256) :: metaname + + call convert_c_string(filename_in,filename) + call convert_c_string(varname_in,varname) + call convert_c_string(metaname_in,metaname) + + metaval_out = get_variable_metadata_double(filename,varname,metaname) + + end function get_variable_metadata_double_c2f +!=====================================================================! + subroutine get_variable_metadata_char_c2f(filename_in, varname_in, metaname_in, metaval_out) bind(c) + use scream_scorpio_interface, only : get_variable_metadata_char + use iso_c_binding, only: C_NULL_CHAR, c_char, c_f_pointer + type(c_ptr), intent(in) :: filename_in + type(c_ptr), intent(in) :: varname_in + type(c_ptr), intent(in) :: metaname_in + type(c_ptr), intent(in) :: metaval_out + + character(len=256) :: filename + character(len=256) :: varname + character(len=256) :: metaname + character(len=256) :: metaval + character(len=256, kind=c_char), pointer :: temp_string + integer :: slen + + call c_f_pointer(metaval_out,temp_string) + + call convert_c_string(filename_in,filename) + call convert_c_string(varname_in,varname) + call convert_c_string(metaname_in,metaname) + + metaval = get_variable_metadata_char(filename,varname,metaname) + + slen = len(trim(metaval)) + ! If string is 255 or less, add terminating char. If not, it's still + ! ok (the C++ string will have length=max_length=256) + if (slen .le. 255) then + temp_string = trim(metaval) // C_NULL_CHAR + else + temp_string = metaval + endif + end subroutine get_variable_metadata_char_c2f !=====================================================================! subroutine register_dimension_c2f(filename_in, shortname_in, longname_in, length, partitioned) bind(c) use scream_scorpio_interface, only : register_dimension @@ -278,6 +388,16 @@ subroutine eam_pio_enddef_c2f(filename_in) bind(c) call convert_c_string(filename_in,filename) call eam_pio_enddef(filename) end subroutine eam_pio_enddef_c2f +!=====================================================================! + subroutine eam_pio_redef_c2f(filename_in) bind(c) + use scream_scorpio_interface, only : eam_pio_redef + type(c_ptr), intent(in) :: filename_in + + character(len=256) :: filename + + call convert_c_string(filename_in,filename) + call eam_pio_redef(filename) + end subroutine eam_pio_redef_c2f !=====================================================================! subroutine convert_c_string(c_string_ptr,f_string) use iso_c_binding, only: c_f_pointer, C_NULL_CHAR diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index ef756345d5e9..21d293dbb15f 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -5,51 +5,68 @@ include(ScreamUtils) include (BuildCprnc) BuildCprnc() +## Test io utils +CreateUnitTest(io_utils "io_utils.cpp" + LIBS scream_io LABELS io + PROPERTIES RESOURCE_LOCK rpointer_file +) + +## Test basic output (no packs, no diags, all avg types, all freq units) +CreateUnitTest(io_basic "io_basic.cpp" + LIBS scream_io LABELS io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} +) + ## Test basic output (no packs, no diags, all avg types, all freq units) -CreateUnitTest(io_basic "io_basic.cpp" "scream_io" LABELS "io" +CreateUnitTest(io_filled "io_filled.cpp" + LIBS scream_io LABELS io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) ## Test packed I/O -CreateUnitTest(io_packed "io_packed.cpp" "scream_io" LABELS "io" +CreateUnitTest(io_packed "io_packed.cpp" + LIBS scream_io LABELS io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) ## Test diagnostic output -CreateUnitTest(io_diags "io_diags.cpp" "scream_io" LABELS "io" +CreateUnitTest(io_diags "io_diags.cpp" + LIBS scream_io LABELS io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) # Test output on SE grid configure_file(io_test_se_grid.yaml io_test_se_grid.yaml) -CreateUnitTest(io_test_se_grid "io_se_grid.cpp" scream_io LABELS "io" +CreateUnitTest(io_test_se_grid "io_se_grid.cpp" + LIBS scream_io LABELS io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) ## Test output restart -# NOTE: Each restart test is a "setup" for the restart_check test, -# and cannot run in parallel with other restart tests, -# due to contention of the rpointer file - -configure_file(io_test_restart.yaml io_test_restart.yaml) -configure_file(io_test_restart_check.yaml io_test_restart_check.yaml) -CreateUnitTest(output_restart_test "output_restart.cpp" scream_io LABELS "io" +# NOTE: These tests cannot run in parallel due to contention of the rpointer file +CreateUnitTest(output_restart "output_restart.cpp" + LIBS scream_io LABELS io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} - PROPERTIES RESOURCE_LOCK rpointer_file FIXTURES_SETUP restart_setup + FIXTURES_SETUP_INDIVIDUAL restart_check_setup + PROPERTIES RESOURCE_LOCK rpointer_file ) -foreach (MPI_RANKS RANGE 1 ${SCREAM_TEST_MAX_RANKS}) - set (SRC_FILE io_output_restart.AVERAGE.nsteps_x10.np${MPI_RANKS}.2000-01-01-00010.nc) - set (TGT_FILE io_output_restart_check.AVERAGE.nsteps_x10.np${MPI_RANKS}.2000-01-01-00010.nc) - add_test (NAME io_test_restart_check_np${MPI_RANKS} - COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_property(TEST io_test_restart_check_np${MPI_RANKS} - PROPERTY FIXTURES_REQUIRED restart_setup) +# For each avg_type and rank combination, compare the monolithic and restared run +include (CompareNCFiles) +foreach (AVG_TYPE IN ITEMS INSTANT AVERAGE) + foreach (MPI_RANKS RANGE 1 ${SCREAM_TEST_MAX_RANKS}) + CompareNCFiles ( + TEST_NAME output_restart_check_${AVG_TYPE}_np${MPI_RANKS} + SRC_FILE monolithic.${AVG_TYPE}.nsteps_x10.np${MPI_RANKS}.2000-01-01-00000.nc + TGT_FILE restarted.${AVG_TYPE}.nsteps_x10.np${MPI_RANKS}.2000-01-01-00000.nc + LABELS io + FIXTURES_REQUIRED restart_check_setup_np${MPI_RANKS}_omp1 + ) + endforeach() endforeach() ## Test remap output -CreateUnitTest(io_remap_test "io_remap_test.cpp" "scream_io;diagnostics" LABELS "io,remap" +CreateUnitTest(io_remap_test "io_remap_test.cpp" + LIBS scream_io diagnostics LABELS io remap MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) - diff --git a/components/eamxx/src/share/io/tests/io_basic.cpp b/components/eamxx/src/share/io/tests/io_basic.cpp index 807607319ec0..4b7ee47bc5f5 100644 --- a/components/eamxx/src/share/io/tests/io_basic.cpp +++ b/components/eamxx/src/share/io/tests/io_basic.cpp @@ -9,6 +9,7 @@ #include "share/field/field.hpp" #include "share/field/field_manager.hpp" +#include "share/util/scream_universal_constants.hpp" #include "share/util/scream_setup_random_test.hpp" #include "share/util/scream_time_stamp.hpp" #include "share/scream_types.hpp" @@ -32,6 +33,7 @@ void add (const Field& f, const double v) { for (int i=0; i get_gm (const ekat::Comm& comm) { - const int nlcols = 3; + // For 2+ ranks tests, this will check IO works correctly + // even if one rank owns 0 dofs + const int ngcols = std::max(comm.size()-1,1); const int nlevs = 4; - const int ngcols = nlcols*comm.size(); - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns",ngcols); - gm_params.set("number_of_vertical_levels",nlevs); - auto gm = create_mesh_free_grids_manager(comm,gm_params); + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,ngcols); gm->build_grids(); return gm; } @@ -105,17 +105,20 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - fm->registration_begins(); - fm->registration_ends(); const auto units = ekat::units::Units::nondimensional(); + int count=0; + using stratts_t = std::map; for (const auto& fl : layouts) { - FID fid("f_"+std::to_string(fl.size()),fl,units,grid->name()); + FID fid("f_"+std::to_string(count),fl,units,grid->name()); Field f(fid); f.allocate_view(); + auto& str_atts = f.get_header().get_extra_data("io: string attributes"); + str_atts["test"] = f.name(); randomize (f,engine,my_pdf); f.get_header().get_tracking().update_time_stamp(t0); fm->add_field(f); + ++count; } return fm; @@ -154,6 +157,11 @@ void write (const std::string& avg_type, const std::string& freq_units, // Create Output manager OutputManager om; + + // Attempt to use invalid fp precision string + om_pl.set("Floating Point Precision",std::string("triple")); + REQUIRE_THROWS (om.setup(comm,om_pl,fm,gm,t0,t0,false)); + om_pl.set("Floating Point Precision",std::string("single")); om.setup(comm,om_pl,fm,gm,t0,t0,false); // Time loop: ensure we always hit 3 output steps @@ -247,6 +255,17 @@ void read (const std::string& avg_type, const std::string& freq_units, } } } + + // Check that the expected metadata was appropriately set for each variable + Real fill_out; + std::string att_test; + for (const auto& fn: fnames) { + scorpio::get_variable_metadata(filename,fn,"_FillValue",fill_out); + REQUIRE(fill_out==constants::DefaultFillValue().value); + + scorpio::get_variable_metadata(filename,fn,"test",att_test); + REQUIRE (att_test==fn); + } } TEST_CASE ("io_basic") { diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index 21603ebc26cc..90abb4c57775 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -104,10 +104,7 @@ get_gm (const ekat::Comm& comm) const int nlcols = 3; const int nlevs = 4; const int ngcols = nlcols*comm.size(); - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns",ngcols); - gm_params.set("number_of_vertical_levels",nlevs); - auto gm = create_mesh_free_grids_manager(comm,gm_params); + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,ngcols); gm->build_grids(); return gm; } @@ -139,8 +136,6 @@ get_fm (const std::shared_ptr& grid, const int nlevs = grid->get_num_vertical_levels(); auto fm = std::make_shared(grid); - fm->registration_begins(); - fm->registration_ends(); const auto units = ekat::units::Units::nondimensional(); FL fl ({COL,LEV}, {nlcols,nlevs}); diff --git a/components/eamxx/src/share/io/tests/io_filled.cpp b/components/eamxx/src/share/io/tests/io_filled.cpp new file mode 100644 index 000000000000..93e14589c44c --- /dev/null +++ b/components/eamxx/src/share/io/tests/io_filled.cpp @@ -0,0 +1,301 @@ +#include + +#include "share/io/scream_output_manager.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_io_utils.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "share/field/field_utils.hpp" +#include "share/field/field.hpp" +#include "share/field/field_manager.hpp" + +#include "share/util/scream_universal_constants.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_time_stamp.hpp" +#include "share/scream_types.hpp" + +#include "ekat/util/ekat_units.hpp" +#include "ekat/ekat_parameter_list.hpp" +#include "ekat/ekat_assert.hpp" +#include "ekat/mpi/ekat_comm.hpp" +#include "ekat/util/ekat_test_utils.hpp" + +#include +#include + +namespace scream { + +constexpr int num_output_steps = 5; +constexpr Real FillValue = constants::DefaultFillValue().value; +constexpr Real fill_threshold = 0.5; + +void set (const Field& f, const double v) { + auto data = f.get_internal_view_data(); + auto nscalars = f.get_header().get_alloc_properties().get_num_scalars(); + for (int i=0; i +get_gm (const ekat::Comm& comm) +{ + const int nlcols = 3; + const int nlevs = 4; + const int ngcols = nlcols*comm.size(); + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,ngcols); + gm->build_grids(); + return gm; +} + +std::shared_ptr +get_fm (const std::shared_ptr& grid, + const util::TimeStamp& t0, const int seed) +{ + using FL = FieldLayout; + using FID = FieldIdentifier; + using namespace ShortFieldTagsNames; + + const int nlcols = grid->get_num_local_dofs(); + const int nlevs = grid->get_num_vertical_levels(); + + std::vector layouts = + { + FL({COL }, {nlcols }), + FL({COL, LEV}, {nlcols, nlevs}), + FL({COL,CMP,ILEV}, {nlcols,2,nlevs+1}) + }; + + auto fm = std::make_shared(grid); + + const auto units = ekat::units::Units::nondimensional(); + for (const auto& fl : layouts) { + FID fid("f_"+std::to_string(fl.size()),fl,units,grid->name()); + Field f(fid); + f.allocate_view(); + f.deep_copy(0.0); // For the "filled" field we start with a filled value. + f.get_header().get_tracking().update_time_stamp(t0); + fm->add_field(f); + } + + return fm; +} + +// Returns fields after initialization +void write (const std::string& avg_type, const std::string& freq_units, + const int freq, const int seed, const ekat::Comm& comm) +{ + // Create grid + auto gm = get_gm(comm); + auto grid = gm->get_grid("Point Grid"); + + // Time advance parameters + auto t0 = get_t0(); + const int dt = get_dt(freq_units); + + // Create some fields + auto fm = get_fm(grid,t0,seed); + std::vector fnames; + for (auto it : *fm) { + fnames.push_back(it.second->name()); + } + + // Create output params + ekat::ParameterList om_pl; + om_pl.set("MPI Ranks in Filename",true); + om_pl.set("filename_prefix",std::string("io_filled")); + om_pl.set("Field Names",fnames); + om_pl.set("Averaging Type", avg_type); + om_pl.set("fill_value",FillValue); + om_pl.set("track_fill",true); + om_pl.set("fill_threshold",fill_threshold); + auto& ctrl_pl = om_pl.sublist("output_control"); + ctrl_pl.set("frequency_units",freq_units); + ctrl_pl.set("Frequency",freq); + ctrl_pl.set("MPI Ranks in Filename",true); + ctrl_pl.set("save_grid_data",false); + + // Create Output manager + OutputManager om; + om.setup(comm,om_pl,fm,gm,t0,t0,false); + + // Time loop: ensure we always hit 3 output steps + const int nsteps = num_output_steps*freq; + auto t = t0; + for (int n=0; nget_field(n); + set(f,setval); + } + + // Run output manager + om.run (t); + } + + // Close file and cleanup + om.finalize(); +} + +void read (const std::string& avg_type, const std::string& freq_units, + const int freq, const int seed, const ekat::Comm& comm) +{ + // Only INSTANT writes at t=0 + bool instant = avg_type=="INSTANT"; + + // Time quantities + auto t0 = get_t0(); + int num_writes = num_output_steps + (instant ? 1 : 0); + + // Get gm + auto gm = get_gm (comm); + auto grid = gm->get_grid("Point Grid"); + + // Get initial fields. Use wrong seed for fm, so fields are not + // inited with right data (avoid getting right answer without reading). + auto fm0 = get_fm(grid,t0,seed); + auto fm = get_fm(grid,t0,-seed-1); + std::vector fnames; + for (auto it : *fm) { + fnames.push_back(it.second->name()); + } + + // Create reader pl + ekat::ParameterList reader_pl; + std::string casename = "io_filled"; + auto filename = casename + + "." + avg_type + + "." + freq_units + + "_x" + std::to_string(freq) + + ".np" + std::to_string(comm.size()) + + "." + t0.to_string() + + ".nc"; + reader_pl.set("Filename",filename); + reader_pl.set("Field Names",fnames); + AtmosphereInput reader(reader_pl,fm); + + // We set the value n to each input field for each odd valued timestep and FillValue for each even valued timestep + // Hence, at output step N = snap*freq, we should get + // avg=INSTANT: output = N if (N%2=0), else Fillvalue + // avg=MAX: output = N if (N%2=0), else N-1 + // avg=MIN: output = N + 1, where n is the first timesnap of the Nth output step. + // we add + 1 more in cases where (N%2=0) because that means the first snap was filled. + // avg=AVERAGE: output = a + M+1 = a + M*(M+1)/M + // The last one comes from + // a + 2*(1 + 2 +..+M)/M = + // a + 2*sum(i)/M = a + 2*(M(M+1)/2)/M, + // where M = freq/2 + ( N%2=0 ? 0 : 1 ), + // a = floor(N/freq)*freq + ( N%2=0 ? 0 : -1) + for (int n=0; nget_field(fn).clone(); + auto f = fm->get_field(fn); + if (avg_type=="MIN") { + Real test_val = ((n+1)*freq%2==0) ? n*freq+1 : n*freq+2; + set(f0,test_val); + REQUIRE (views_are_equal(f,f0)); + } else if (avg_type=="MAX") { + Real test_val = ((n+1)*freq%2==0) ? (n+1)*freq : (n+1)*freq-1; + set(f0,test_val); + REQUIRE (views_are_equal(f,f0)); + } else if (avg_type=="INSTANT") { + Real test_val = (n*freq%2==0) ? n*freq : FillValue; + set(f0,test_val); + REQUIRE (views_are_equal(f,f0)); + } else { // Is avg_type = AVERAGE + // Note, for AVERAGE type output with filling we need to check that the + // number of contributing fill steps surpasses the fill_threshold, if not + // then we know that the snap will reflect the fill value. + Real test_val; + Real M = freq/2 + (n%2==0 ? 0.0 : 1.0); + Real a = n*freq + (n%2==0 ? 0.0 : -1.0); + test_val = (M/freq > fill_threshold) ? a + (M+1.0) : FillValue; + set(f0,test_val); + REQUIRE (views_are_equal(f,f0)); + } + } + } + + // Check that the fill value gets appropriately set for each variable + Real fill_out; + for (const auto& fn: fnames) { + scorpio::get_variable_metadata(filename,fn,"_FillValue",fill_out); + REQUIRE(fill_out==constants::DefaultFillValue().value); + } +} + +TEST_CASE ("io_filled") { + std::vector freq_units = { + "nsteps", + "nsecs", + "nmins", + "nhours", + "ndays" + }; + std::vector avg_type = { + "INSTANT", + "MAX", + "MIN", + "AVERAGE" + }; + + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); + + auto seed = get_random_test_seed(&comm); + + const int freq = 5; + auto print = [&] (const std::string& s, int line_len = -1) { + if (comm.am_i_root()) { + if (line_len<0) { + std::cout << s; + } else { + std::cout << std::left << std::setw(line_len) << std::setfill('.') << s; + } + } + }; + + for (const auto& units : freq_units) { + print ("-> Output frequency: " + units + "\n"); + for (const auto& avg : avg_type) { + print(" -> Averaging type: " + avg + " ", 40); + write(avg,units,freq,seed,comm); + read(avg,units,freq,seed,comm); + print(" PASS\n"); + } + } + scorpio::eam_pio_finalize(); +} + +} // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_packed.cpp b/components/eamxx/src/share/io/tests/io_packed.cpp index c16185b93cec..f4b467c619c1 100644 --- a/components/eamxx/src/share/io/tests/io_packed.cpp +++ b/components/eamxx/src/share/io/tests/io_packed.cpp @@ -42,10 +42,7 @@ get_gm (const ekat::Comm& comm) const int nlcols = 3; const int nlevs = 16; const int ngcols = nlcols*comm.size(); - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns",ngcols); - gm_params.set("number_of_vertical_levels",nlevs); - auto gm = create_mesh_free_grids_manager(comm,gm_params); + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,ngcols); gm->build_grids(); return gm; } @@ -82,8 +79,6 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - fm->registration_begins(); - fm->registration_ends(); const auto units = ekat::units::Units::nondimensional(); for (const auto& fl : layouts) { diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index a2d846a70949..91fa95b3fc29 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -15,6 +15,7 @@ using namespace scream; constexpr int packsize = SCREAM_SMALL_PACK_SIZE; using Pack = ekat::Pack; +using stratts_t = std::map; Real set_pressure(const Real p_top, const Real p_bot, const int nlevs, const int level); @@ -110,12 +111,12 @@ TEST_CASE("io_remap_test","io_remap_test") scorpio::register_dimension(remap_filename,"n_a", "n_a", ncols_src, true); scorpio::register_dimension(remap_filename,"n_b", "n_b", ncols_tgt, true); scorpio::register_dimension(remap_filename,"n_s", "n_s", ncols_src, true); - scorpio::register_dimension(remap_filename,"nlevs", "nlevs", nlevs_tgt, false); + scorpio::register_dimension(remap_filename,"lev", "lev", nlevs_tgt, false); scorpio::register_variable(remap_filename,"col","col","none",{"n_s"},"real","int","int-nnz"); scorpio::register_variable(remap_filename,"row","row","none",{"n_s"},"real","int","int-nnz"); scorpio::register_variable(remap_filename,"S","S","none",{"n_s"},"real","real","Real-nnz"); - scorpio::register_variable(remap_filename,"p_levs","p_levs","none",{"nlevs"},"real","real","Real-nlevs"); + scorpio::register_variable(remap_filename,"p_levs","p_levs","none",{"lev"},"real","real","Real-lev"); scorpio::set_dof(remap_filename,"col",dofs_cols.size(),dofs_cols.data()); scorpio::set_dof(remap_filename,"row",dofs_cols.size(),dofs_cols.data()); @@ -271,10 +272,12 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> Test Remapped Output ... \n",io_comm); // ------------------------------------------------------------------------------------------------------ // --- Vertical Remapping --- + std::vector fnames = {"Y_flat","Y_mid","Y_int","V_mid","V_int"}; + { // Note, the vertical remapper defaults to a mask value of std numeric limits scaled by 0.1; const float mask_val = vert_remap_control.isParameter("Fill Value") - ? vert_remap_control.get("Fill Value") : DEFAULT_FILL_VALUE; + ? vert_remap_control.get("Fill Value") : constants::DefaultFillValue().value; print (" -> vertical remap ... \n",io_comm); auto gm_vert = get_test_gm(io_comm,ncols_src,nlevs_tgt); auto grid_vert = gm_vert->get_grid("Point Grid"); @@ -282,6 +285,20 @@ TEST_CASE("io_remap_test","io_remap_test") auto vert_in = set_input_params("remap_vertical",io_comm,t0.to_string(),p_ref); AtmosphereInput test_input(vert_in,fm_vert); test_input.read_variables(); + + // Check the "test" metadata, which should match the field name + // Note: the FieldAtPressureLevel diag should get the attribute from its input field, + // so the valuf for "Y_int"_at_XPa should be "Y_int" + std::string att_val; + const auto& filename = vert_in.get("Filename"); + for (auto& fname : fnames) { + scorpio::get_variable_metadata(filename,fname,"test",att_val); + REQUIRE (att_val==fname); + } + std::string f_at_lev_name = "Y_int_at_" + std::to_string(p_ref) + "Pa"; + scorpio::get_variable_metadata(filename,f_at_lev_name,"test",att_val); + REQUIRE (att_val=="Y_int"); + test_input.finalize(); // Test vertically remapped output. @@ -292,7 +309,7 @@ TEST_CASE("io_remap_test","io_remap_test") // // NOTE: For scorpio_output.cpp the mask value for vertical remapping is std::numeric_limits::max()/10.0 const auto& Yf_f_vert = fm_vert->get_field("Y_flat"); - const auto& Ys_f_vert = fm_vert->get_field("Y_int@"+std::to_string(p_ref)+"Pa"); + const auto& Ys_f_vert = fm_vert->get_field("Y_int_at_"+std::to_string(p_ref)+"Pa"); const auto& Ym_f_vert = fm_vert->get_field("Y_mid"); const auto& Yi_f_vert = fm_vert->get_field("Y_int"); const auto& Vm_f_vert = fm_vert->get_field("V_mid"); @@ -330,7 +347,7 @@ TEST_CASE("io_remap_test","io_remap_test") { // Note, the vertical remapper defaults to a mask value of std numeric limits scaled by 0.1; const float mask_val = horiz_remap_control.isParameter("Fill Value") - ? horiz_remap_control.get("Fill Value") : DEFAULT_FILL_VALUE; + ? horiz_remap_control.get("Fill Value") : constants::DefaultFillValue().value; print (" -> horizontal remap ... \n",io_comm); auto gm_horiz = get_test_gm(io_comm,ncols_tgt,nlevs_src); auto grid_horiz = gm_horiz->get_grid("Point Grid"); @@ -338,16 +355,29 @@ TEST_CASE("io_remap_test","io_remap_test") auto horiz_in = set_input_params("remap_horizontal",io_comm,t0.to_string(),p_ref); AtmosphereInput test_input(horiz_in,fm_horiz); test_input.read_variables(); + + // Check the "test" metadata, which should match the field name + // Note: the FieldAtPressureLevel diag should get the attribute from its input field, + // so the valuf for "Y_int"_at_XPa should be "Y_int" + std::string att_val; + const auto& filename = horiz_in.get("Filename"); + for (auto& fname : fnames) { + scorpio::get_variable_metadata(filename,fname,"test",att_val); + REQUIRE (att_val==fname); + } + std::string f_at_lev_name = "Y_int_at_" + std::to_string(p_ref) + "Pa"; + scorpio::get_variable_metadata(filename,f_at_lev_name,"test",att_val); + REQUIRE (att_val=="Y_int"); test_input.finalize(); // Test horizontally remapped output. // The remap we are testing is rather simple, each pair of subsequent columns are remapped to a single // column using `wgt` and `1-wgt` respectively. // - // Note: For horizontal remapping we added the variable Y_min@XPa to check that this diagnostic does + // Note: For horizontal remapping we added the variable Y_min_at_XPa to check that this diagnostic does // provide some masking, since it applies vertical remapping. const auto& Yf_f_horiz = fm_horiz->get_field("Y_flat"); - const auto& Ys_f_horiz = fm_horiz->get_field("Y_int@"+std::to_string(p_ref)+"Pa"); + const auto& Ys_f_horiz = fm_horiz->get_field("Y_int_at_"+std::to_string(p_ref)+"Pa"); const auto& Ym_f_horiz = fm_horiz->get_field("Y_mid"); const auto& Yi_f_horiz = fm_horiz->get_field("Y_int"); const auto& Vm_f_horiz = fm_horiz->get_field("V_mid"); @@ -403,7 +433,7 @@ TEST_CASE("io_remap_test","io_remap_test") // --- Vertical + Horizontal Remapping --- { const float mask_val = vert_horiz_remap_control.isParameter("Fill Value") - ? vert_horiz_remap_control.get("Fill Value") : DEFAULT_FILL_VALUE; + ? vert_horiz_remap_control.get("Fill Value") : constants::DefaultFillValue().value; print (" -> vertical + horizontal remap ... \n",io_comm); auto gm_vh = get_test_gm(io_comm,ncols_tgt,nlevs_tgt); auto grid_vh = gm_vh->get_grid("Point Grid"); @@ -411,6 +441,19 @@ TEST_CASE("io_remap_test","io_remap_test") auto vh_in = set_input_params("remap_vertical_horizontal",io_comm,t0.to_string(),p_ref); AtmosphereInput test_input(vh_in,fm_vh); test_input.read_variables(); + + // Check the "test" metadata, which should match the field name + // Note: the FieldAtPressureLevel diag should get the attribute from its input field, + // so the valuf for "Y_int"_at_XPa should be "Y_int" + std::string att_val; + const auto& filename = vh_in.get("Filename"); + for (auto& fname : fnames) { + scorpio::get_variable_metadata(filename,fname,"test",att_val); + REQUIRE (att_val==fname); + } + std::string f_at_lev_name = "Y_int_at_" + std::to_string(p_ref) + "Pa"; + scorpio::get_variable_metadata(filename,f_at_lev_name,"test",att_val); + REQUIRE (att_val=="Y_int"); test_input.finalize(); // Test vertically + horizontally remapped output. @@ -418,11 +461,11 @@ TEST_CASE("io_remap_test","io_remap_test") // There should be maksing in the vertical in all locations where the target pressure // is lower higher than the surface pressure, just like in the vertical test. This should // also translate to more masking in the horizontal reamapping. So we must check for potential - // masking for all variables rather than just the Y_int@XPa variable for the horizontal interpolation. + // masking for all variables rather than just the Y_int_at_XPa variable for the horizontal interpolation. // // NOTE: For scorpio_output.cpp the mask value for vertical remapping is std::numeric_limits::max()/10.0 const auto& Yf_f_vh = fm_vh->get_field("Y_flat"); - const auto& Ys_f_vh = fm_vh->get_field("Y_int@"+std::to_string(p_ref)+"Pa"); + const auto& Ys_f_vh = fm_vh->get_field("Y_int_at_"+std::to_string(p_ref)+"Pa"); const auto& Ym_f_vh = fm_vh->get_field("Y_mid"); const auto& Yi_f_vh = fm_vh->get_field("Y_int"); const auto& Vm_f_vh = fm_vh->get_field("V_mid"); @@ -524,10 +567,7 @@ Real calculate_output(const Real pressure, const int col, const int cmp) /*==========================================================================================================*/ std::shared_ptr get_test_gm(const ekat::Comm& io_comm, const Int num_gcols, const Int num_levs) { - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns",num_gcols); - gm_params.set("number_of_vertical_levels",num_levs); - auto gm = create_mesh_free_grids_manager(io_comm,gm_params); + auto gm = create_mesh_free_grids_manager(io_comm,0,0,num_levs,num_gcols); gm->build_grids(); return gm; } @@ -584,7 +624,7 @@ std::shared_ptr get_test_fm(std::shared_ptr gr fm->register_field(FR{fid_Vm,"output",Pack::n}); fm->register_field(FR{fid_Vi,"output",Pack::n}); if (p_ref>=0) { - FieldIdentifier fid_di("Y_int@"+std::to_string(p_ref)+"Pa", FL{tag_h,dims_h},m,gn); + FieldIdentifier fid_di("Y_int_at_"+std::to_string(p_ref)+"Pa", FL{tag_h,dims_h},m,gn); fm->register_field(FR{fid_di,"output"}); } fm->registration_ends(); @@ -599,7 +639,13 @@ std::shared_ptr get_test_fm(std::shared_ptr gr auto f_Vm = fm->get_field(fid_Vm); auto f_Vi = fm->get_field(fid_Vi); - // Update timestamp + // Set some string to be written to file as attribute to the variables + for (const std::string& fname : {"Y_flat","Y_mid","Y_int","V_mid","V_int"}) { + auto& f = fm->get_field(fname); + auto& str_atts = f.get_header().get_extra_data("io: string attributes"); + str_atts["test"] = fname; + } + // Update timestamp util::TimeStamp time ({2000,1,1},{0,0,0}); fm->init_fields_time_stamp(time); @@ -613,7 +659,7 @@ std::shared_ptr get_test_fm(std::shared_ptr gr f_Vm.sync_to_dev(); f_Vi.sync_to_dev(); if (p_ref>=0) { - auto f_di = fm->get_field("Y_int@"+std::to_string(p_ref)+"Pa"); + auto f_di = fm->get_field("Y_int_at_"+std::to_string(p_ref)+"Pa"); f_di.sync_to_dev(); } @@ -636,7 +682,7 @@ ekat::ParameterList set_output_params(const std::string& name, const std::string vos_type fields_out = {"Y_flat", "Y_mid", "Y_int", "V_mid", "V_int"}; if (p_ref>=0) { - fields_out.push_back("Y_int@"+std::to_string(p_ref)+"Pa"); + fields_out.push_back("Y_int_at_"+std::to_string(p_ref)+"Pa"); } if (!vert_remap && !horiz_remap) { fields_out.push_back("p_surf"); @@ -663,7 +709,7 @@ ekat::ParameterList set_input_params(const std::string& name, ekat::Comm& comm, in_params.set("Filename",filename); vos_type fields_in = {"Y_flat", "Y_mid", "Y_int", "V_mid", "V_int"}; if (p_ref>=0) { - fields_in.push_back("Y_int@"+std::to_string(p_ref)+"Pa"); + fields_in.push_back("Y_int_at_"+std::to_string(p_ref)+"Pa"); } in_params.set("Field Names", fields_in); diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index 08cb20727c0a..16872a7491f8 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -152,12 +152,7 @@ get_test_fm(const std::shared_ptr& grid, std::shared_ptr get_test_gm(const ekat::Comm& io_comm, const int num_my_elems, const int np, const int num_levs) { - ekat::ParameterList gm_params; - gm_params.set("number_of_local_elements",num_my_elems); - gm_params.set("number_of_gauss_points",np); - gm_params.set("number_of_vertical_levels",num_levs); - - auto gm = create_mesh_free_grids_manager(io_comm,gm_params); + auto gm = create_mesh_free_grids_manager(io_comm,num_my_elems,np,num_levs,0); gm->build_grids(); return gm; diff --git a/components/eamxx/src/share/io/tests/io_test_restart.yaml b/components/eamxx/src/share/io/tests/io_test_restart.yaml deleted file mode 100644 index fb4ba678884c..000000000000 --- a/components/eamxx/src/share/io/tests/io_test_restart.yaml +++ /dev/null @@ -1,15 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: io_output_restart -Averaging Type: Average -Max Snapshots Per File: 1 -Field Names: [field_1, field_2, field_3, field_4] -output_control: - MPI Ranks in Filename: true - Frequency: 10 - frequency_units: nsteps -Checkpoint Control: - MPI Ranks in Filename: true - Frequency: 5 - frequency_units: nsteps -... diff --git a/components/eamxx/src/share/io/tests/io_test_restart_check.yaml b/components/eamxx/src/share/io/tests/io_test_restart_check.yaml deleted file mode 100644 index 3c05931db503..000000000000 --- a/components/eamxx/src/share/io/tests/io_test_restart_check.yaml +++ /dev/null @@ -1,14 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: io_output_restart_check -Averaging Type: Average -Max Snapshots Per File: 1 -Field Names: [field_1, field_2, field_3, field_4] -output_control: - MPI Ranks in Filename: true - Frequency: 10 - frequency_units: nsteps -Restart: - MPI Ranks in Filename: true - filename_prefix: io_output_restart -... diff --git a/components/eamxx/src/share/io/tests/io_utils.cpp b/components/eamxx/src/share/io/tests/io_utils.cpp new file mode 100644 index 000000000000..6064a29567a0 --- /dev/null +++ b/components/eamxx/src/share/io/tests/io_utils.cpp @@ -0,0 +1,123 @@ +#include + +#include +#include + +#include + +TEST_CASE ("find_filename_in_rpointer") { + using namespace scream; + + ekat::Comm comm(MPI_COMM_WORLD); + + util::TimeStamp t0({2023,9,7},{12,0,0}); + util::TimeStamp t1({2023,9,7},{13,0,0}); + + // Create a dummy rpointer + std::ofstream rpointer ("rpointer.atm"); + + rpointer << "foo.r." + t0.to_string() + ".nc\n"; + rpointer << "bar2.rhist." + t0.to_string() + ".nc\n"; + rpointer << "bar.rhist." + t0.to_string() + ".nc\n"; + rpointer.close(); + + // Now test find_filename_in_rpointer with different inputs + + REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0)); // wrong prefix + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t1)); // wrong timestamp + REQUIRE_THROWS (find_filename_in_rpointer("bar",true, comm,t0)); // bar is not model restart + REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0)); // foo is model restart + + REQUIRE (find_filename_in_rpointer("bar", false,comm,t0)==("bar.rhist."+t0.to_string()+".nc")); + REQUIRE (find_filename_in_rpointer("bar2",false,comm,t0)==("bar2.rhist."+t0.to_string()+".nc")); + REQUIRE (find_filename_in_rpointer("foo", true, comm,t0)==("foo.r."+t0.to_string()+".nc")); +} + +TEST_CASE ("io_control") { + using namespace scream; + + util::TimeStamp t0({2023,9,7},{12,0,0}); + + IOControl control; + control.frequency = 2; + control.timestamp_of_last_write = t0; + + SECTION ("none") { + control.frequency_units = "none"; + REQUIRE (not control.output_enabled()); + REQUIRE (not control.is_write_step(t0)); + } + + SECTION ("never") { + control.frequency_units = "never"; + REQUIRE (not control.output_enabled()); + REQUIRE (not control.is_write_step(t0)); + } + + SECTION ("nsteps") { + control.frequency_units = "nsteps"; + auto t1 = t0 + 1; + auto t2 = t1 + 1; + REQUIRE (control.output_enabled()); + REQUIRE (not control.is_write_step(t1)); + REQUIRE (control.is_write_step(t2)); + } + + SECTION ("nsecs") { + control.frequency_units = "nsecs"; + auto t1 = t0 + 1; + auto t2 = t1 + 1; + REQUIRE (control.output_enabled()); + REQUIRE (not control.is_write_step(t1)); + REQUIRE (control.is_write_step(t2)); + } + + SECTION ("nmins") { + control.frequency_units = "nmins"; + auto t1 = t0 + 60; + auto t2 = t1 + 60; + REQUIRE (control.output_enabled()); + REQUIRE (not control.is_write_step(t1)); + REQUIRE (control.is_write_step(t2)); + } + + SECTION ("nhours") { + control.frequency_units = "nhours"; + auto t1 = t0 + 3600; + auto t2 = t1 + 3600; + REQUIRE (control.output_enabled()); + REQUIRE (not control.is_write_step(t1)); + REQUIRE (control.is_write_step(t2)); + } + + SECTION ("ndays") { + control.frequency_units = "ndays"; + auto t1 = t0 + 86400; + auto t2 = t1 + 86400; + REQUIRE (control.output_enabled()); + REQUIRE (not control.is_write_step(t1)); + REQUIRE (control.is_write_step(t2)); + } + + SECTION ("nmonths") { + control.frequency_units = "nmonths"; + util::TimeStamp t1({2023,10,7},{12,0,0}); + util::TimeStamp t2({2023,11,7},{12,0,0}); + util::TimeStamp t3({2023,11,7},{13,0,0}); + REQUIRE (control.output_enabled()); + REQUIRE (not control.is_write_step(t1)); + REQUIRE (control.is_write_step(t2)); + REQUIRE (not control.is_write_step(t3)); + } + + SECTION ("nyears") { + control.frequency_units = "nyears"; + util::TimeStamp t1({2024,9,7},{12,0,0}); + util::TimeStamp t2({2025,9,7},{12,0,0}); + util::TimeStamp t3({2025,9,7},{13,0,0}); + REQUIRE (control.output_enabled()); + REQUIRE (not control.is_write_step(t1)); + REQUIRE (control.is_write_step(t2)); + REQUIRE (not control.is_write_step(t3)); + } +} diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index 50899fa5566c..118b9a84836a 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -18,19 +18,25 @@ #include "share/scream_types.hpp" #include "ekat/ekat_parameter_list.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" #include "ekat/util/ekat_string_utils.hpp" +#include "ekat/util/ekat_test_utils.hpp" #include +#include #include namespace scream { +constexpr Real FillValue = constants::DefaultFillValue().value; + +std::shared_ptr +get_test_fm(const std::shared_ptr& grid); + std::shared_ptr -get_test_fm(std::shared_ptr grid); +clone_fm (const std::shared_ptr& fm); std::shared_ptr -get_test_gm(const ekat::Comm& io_comm, const Int num_gcols, const Int num_levs); +get_test_gm(const ekat::Comm& comm, const Int num_gcols, const Int num_levs); template void randomize_fields (const FieldManager& fm, Engine& engine); @@ -39,125 +45,118 @@ void time_advance (const FieldManager& fm, const std::list& fnames, const int dt); -std::shared_ptr -backup_fm (const std::shared_ptr& src_fm); - TEST_CASE("output_restart","io") { - // Note to AaronDonahue: You are trying to figure out why you can't change the number of cols and levs for this test. - // Something having to do with freeing up and then resetting the io_decompositions. - ekat::Comm io_comm(MPI_COMM_WORLD); - Int num_gcols = 2*io_comm.size(); - Int num_levs = 3; + ekat::Comm comm(MPI_COMM_WORLD); + + // If running with 2+ ranks, this will check that restart works correctly + // even if some ranks own no dofs + int num_gcols = std::max(comm.size()-1,1); + int num_levs = 3; + int dt = 1; - auto engine = setup_random_test(&io_comm); + auto engine = setup_random_test(&comm); // First set up a field manager and grids manager to interact with the output functions - auto gm = get_test_gm(io_comm,num_gcols,num_levs); + auto gm = get_test_gm(comm,num_gcols,num_levs); auto grid = gm->get_grid("Point Grid"); - auto field_manager = get_test_fm(grid); - randomize_fields(*field_manager,engine); - const auto& out_fields = field_manager->get_groups_info().at("output")->m_fields_names; + + // The the IC field manager + auto fm0 = get_test_fm(grid); + randomize_fields(*fm0,engine); + + const auto& out_fields = fm0->get_groups_info().at("output")->m_fields_names; // Initialize the pio_subsystem for this test: - MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); + MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); scorpio::eam_init_pio_subsystem(fcomm); // Timestamp of the simulation initial time util::TimeStamp t0 ({2000,1,1},{0,0,0}); - // Create an Output manager for testing output - std::string param_filename = "io_test_restart.yaml"; + // Create output params (some options are set below, depending on the run type ekat::ParameterList output_params; - ekat::parse_yaml_file(param_filename,output_params); output_params.set("Floating Point Precision","real"); - OutputManager output_manager; - output_manager.setup(io_comm,output_params,field_manager,gm,t0,t0,false); - - // We advance the fields, by adding dt to each entry of the fields at each time step - // The output restart data is written every 5 time steps, while the output freq is 10. - // We run for 15 steps, which means that after 15 steps we should have a history restart - // file, with output history right in the middle between two output steps. - - // Time-advance all fields - const int dt = 1; - const int nsteps = 15; - auto time = t0; - for (int i=0; i> line) { - rpointer_content += line + "\n"; + output_params.set>("Field Names",{"field_1", "field_2", "field_3", "field_4","field_5"}); + output_params.set("fill_value",FillValue); + output_params.sublist("output_control").set("MPI Ranks in Filename","true"); + output_params.sublist("output_control").set("frequency_units","nsteps"); + output_params.sublist("output_control").set("Frequency",10); + output_params.sublist("Checkpoint Control").set("MPI Ranks in Filename","true"); + output_params.sublist("Checkpoint Control").set("Frequency",5); + // This skips a test that only matters for AD runs + output_params.sublist("Checkpoint Control").set("is_unit_testing","true"); + output_params.sublist("Restart").set("MPI Ranks in Filename","true"); + + // Creates and runs an OM from output_params and given inputs + auto run = [&](std::shared_ptr fm, + const util::TimeStamp& case_t0, + const util::TimeStamp& run_t0, + const int nsteps) + { + OutputManager output_manager; + output_manager.setup(comm,output_params,fm,gm,run_t0,case_t0,false); + + // We advance the fields, by adding dt to each entry of the fields at each time step + // The output restart data is written every 5 time steps, while the output freq is 10. + auto time = run_t0; + for (int i=0; i("Floating Point Precision","real"); - - OutputManager output_manager_res; - output_manager_res.setup(io_comm,output_params_res,fm_res,gm,time_res,t0,false); - - // Run 5 more steps from the restart, to get to the next output step. - // We should be generating the same output file as before. - for (int i=0; i<5; ++i) { - time_advance(*fm_res,out_fields,dt); - time_res += dt; - output_manager_res.run(time_res); + output_manager.finalize(); + }; + + auto print = [&] (const std::string& s, int line_len = -1) { + if (comm.am_i_root()) { + if (line_len<0) { + std::cout << s; + } else { + std::cout << std::left << std::setw(line_len) << std::setfill('.') << s; + } + } + }; + // Run test for different avg type choices + for (const std::string& avg_type : {"INSTANT","AVERAGE"}) { + { + // In normal runs, the OM for the model restart takes care of nuking rpointer.atm, + // and re-creating a new one. Here, we don't have that, so we must nuke it manually + std::ofstream ofs; + ofs.open("rpointer.atm", std::ofstream::out | std::ofstream::trunc); + } + print(" -> Averaging type: " + avg_type + " ", 40); + output_params.set("Averaging Type",avg_type); + + // 1. Run for full 20 days, no restarts needed + auto fm_mono = clone_fm(fm0); + output_params.set("filename_prefix","monolithic"); + output_params.sublist("Checkpoint Control").set("frequency_units","never"); + run(fm_mono,t0,t0,20); + + // 2. Run for 15 days on fm0, write restart every 5 steps + auto fm_rest = clone_fm(fm0); + output_params.set("filename_prefix","restarted"); + output_params.sublist("Checkpoint Control").set("frequency_units","nsteps"); + run(fm_rest,t0,t0,15); + + // 3. Restart the second run at step=15, and do 5 more steps + // NOTE: keep fm_rest FM, since we are not testing the restart of the state, just the history. + // Here, we proceed as if the AD already restarted the state correctly. + output_params.sublist("Checkpoint Control").set("frequency_units","never"); + + // Ensure nsteps is equal to 15 upon restart + auto run_t0 = (t0+15*dt).clone(15); + run(fm_rest,t0,run_t0,5); + print(" DONE\n"); } - output_manager_res.finalize(); - // Finalize everything scorpio::eam_pio_finalize(); } /*=============================================================================================*/ -std::shared_ptr get_test_fm(std::shared_ptr grid) +std::shared_ptr +get_test_fm(const std::shared_ptr& grid) { using namespace ShortFieldTagsNames; using namespace ekat::units; @@ -177,11 +176,13 @@ std::shared_ptr get_test_fm(std::shared_ptr gr std::vector tag_v = {LEV}; std::vector tag_2d = {COL,LEV}; std::vector tag_3d = {COL,CMP,LEV}; + std::vector tag_bnd = {COL,SWBND,LEV}; std::vector dims_h = {num_lcols}; std::vector dims_v = {num_levs}; std::vector dims_2d = {num_lcols,num_levs}; std::vector dims_3d = {num_lcols,2,num_levs}; + std::vector dims_bnd = {num_lcols,3,num_levs}; const std::string& gn = grid->name(); @@ -189,6 +190,7 @@ std::shared_ptr get_test_fm(std::shared_ptr gr FieldIdentifier fid2("field_2",FL{tag_v,dims_v},kg,gn); FieldIdentifier fid3("field_3",FL{tag_2d,dims_2d},kg/m,gn); FieldIdentifier fid4("field_4",FL{tag_3d,dims_3d},kg/m,gn); + FieldIdentifier fid5("field_5",FL{tag_bnd,dims_bnd},m*m,gn); // Register fields with fm fm->registration_begins(); @@ -196,12 +198,13 @@ std::shared_ptr get_test_fm(std::shared_ptr gr fm->register_field(FR{fid2,SL{"output"}}); fm->register_field(FR{fid3,SL{"output"}}); fm->register_field(FR{fid4,SL{"output"}}); + fm->register_field(FR{fid5,SL{"output"}}); fm->registration_ends(); // Initialize fields to -1.0, and set initial time stamp util::TimeStamp time ({2000,1,1},{0,0,0}); fm->init_fields_time_stamp(time); - for (const auto& fn : {"field_1","field_2","field_3","field_4"} ) { + for (const auto& fn : {"field_1","field_2","field_3","field_4","field_5"} ) { fm->get_field(fn).deep_copy(-1.0); fm->get_field(fn).sync_to_host(); } @@ -209,6 +212,18 @@ std::shared_ptr get_test_fm(std::shared_ptr gr return fm; } +std::shared_ptr +clone_fm(const std::shared_ptr& src) { + auto copy = std::make_shared(src->get_grid()); + copy->registration_begins(); + copy->registration_ends(); + for (auto it : *src) { + copy->add_field(it.second->clone()); + } + + return copy; +} + /*=================================================================================================*/ template void randomize_fields (const FieldManager& fm, Engine& engine) @@ -221,20 +236,19 @@ void randomize_fields (const FieldManager& fm, Engine& engine) const auto& f2 = fm.get_field("field_2"); const auto& f3 = fm.get_field("field_3"); const auto& f4 = fm.get_field("field_4"); + const auto& f5 = fm.get_field("field_5"); randomize(f1,engine,pdf); randomize(f2,engine,pdf); randomize(f3,engine,pdf); randomize(f4,engine,pdf); + randomize(f5,engine,pdf); } /*=============================================================================================*/ std::shared_ptr -get_test_gm(const ekat::Comm& io_comm, const Int num_gcols, const Int num_levs) +get_test_gm(const ekat::Comm& comm, const Int num_gcols, const Int num_levs) { - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns",num_gcols); - gm_params.set("number_of_vertical_levels",num_levs); - auto gm = create_mesh_free_grids_manager(io_comm,gm_params); + auto gm = create_mesh_free_grids_manager(comm,0,0,num_levs,num_gcols); gm->build_grids(); return gm; } @@ -271,7 +285,14 @@ void time_advance (const FieldManager& fm, for (int i=0; i& src_fm) { // Now, create a copy of the field manager current status, for comparisong auto dst_fm = get_test_fm(src_fm->get_grid()); - for (const auto& fn : {"field_1","field_2","field_3","field_4"} ) { + for (const auto& fn : {"field_1","field_2","field_3","field_4","field_5"} ) { auto f_dst = dst_fm->get_field(fn); const auto f_src = src_fm->get_field(fn); f_dst.deep_copy(f_src); diff --git a/components/eamxx/src/share/property_checks/field_nan_check.cpp b/components/eamxx/src/share/property_checks/field_nan_check.cpp index 241e905b1667..ba120ba4c5eb 100644 --- a/components/eamxx/src/share/property_checks/field_nan_check.cpp +++ b/components/eamxx/src/share/property_checks/field_nan_check.cpp @@ -138,7 +138,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check_impl() const { col_lid = indices[0]; auto gids = m_grid->get_dofs_gids().get_view(); - res_and_msg.msg += " - entry (" + std::to_string(gids(col_lid));; + res_and_msg.msg += " - indices (w/ global column index): (" + std::to_string(gids(col_lid)); for (size_t i=1; iget_geometry_data("lon").get_internal_view_data(); res_and_msg.msg += " - lat/lon: (" + std::to_string(lat[col_lid]) + ", " + std::to_string(lon[col_lid]) + ")\n"; } + bool has_additional_col_info = not additional_data_fields().empty(); + if (has_additional_col_info) { + std::stringstream msg; + msg << " - additional data (w/ local column index):\n"; + for (auto& f : additional_data_fields()) { + f.sync_to_host(); + msg << "\n"; + print_field_hyperslab(f, {COL}, {col_lid}, msg); + } + msg << "\n END OF ADDITIONAL DATA\n"; + res_and_msg.msg += msg.str(); + } } + } else { res_and_msg.msg = "FieldNaNCheck passed.\n"; } diff --git a/components/eamxx/src/share/property_checks/field_within_interval_check.cpp b/components/eamxx/src/share/property_checks/field_within_interval_check.cpp index c17225e1e593..bd436bb192a4 100644 --- a/components/eamxx/src/share/property_checks/field_within_interval_check.cpp +++ b/components/eamxx/src/share/property_checks/field_within_interval_check.cpp @@ -1,5 +1,6 @@ #include "share/property_checks/field_within_interval_check.hpp" #include "share/util/scream_array_utils.hpp" +#include "share/field/field_utils.hpp" #include @@ -229,12 +230,13 @@ PropertyCheck::ResultAndMsg FieldWithinIntervalCheck::check_impl () const res_and_msg.msg = "Check passed.\n"; res_and_msg.msg += " - check name:" + this->name() + "\n"; res_and_msg.msg += " - field id: " + f.get_header().get_identifier().get_id_string() + "\n"; - } else { - res_and_msg.msg = "Check failed.\n"; - res_and_msg.msg += " - check name: " + this->name() + "\n"; - res_and_msg.msg += " - field id: " + f.get_header().get_identifier().get_id_string() + "\n"; + return res_and_msg; } + res_and_msg.msg = "Check failed.\n"; + res_and_msg.msg += " - check name: " + this->name() + "\n"; + res_and_msg.msg += " - field id: " + f.get_header().get_identifier().get_id_string() + "\n"; + auto idx_min = unflatten_idx(layout.dims(),minmaxloc.min_loc); auto idx_max = unflatten_idx(layout.dims(),minmaxloc.max_loc); @@ -248,11 +250,12 @@ PropertyCheck::ResultAndMsg FieldWithinIntervalCheck::check_impl () const using namespace ShortFieldTagsNames; - int min_col_lid, max_col_lid; - bool has_latlon; + int min_col_lid = -1, max_col_lid = -1; + bool has_latlon = false; bool has_col_info = m_grid and layout.tag(0)==COL; - const Real* lat; - const Real* lon; + bool has_additional_col_info = not additional_data_fields().empty(); + const Real* lat = nullptr; + const Real* lon = nullptr; if (has_col_info) { // We are storing grid info, and the field is over columns. Get col id and coords. @@ -270,7 +273,7 @@ PropertyCheck::ResultAndMsg FieldWithinIntervalCheck::check_impl () const msg << " - value: " << minmaxloc.min_val << "\n"; if (has_col_info) { auto gids = m_grid->get_dofs_gids().get_view(); - msg << " - entry: (" << gids(min_col_lid); + msg << " - indices (w/ global column index): (" << gids(min_col_lid); for (size_t i=1; iget_dofs_gids().get_view(); - msg << " - entry: (" << gids(max_col_lid); + msg << " - indices (w/ global column index): (" << gids(max_col_lid); for (size_t i=1; i namespace scream @@ -7,21 +9,21 @@ namespace scream MassAndEnergyColumnConservationCheck:: MassAndEnergyColumnConservationCheck (const std::shared_ptr& grid, - const Real mass_error_tolerance, - const Real energy_error_tolerance, - const std::shared_ptr& pseudo_density_ptr, - const std::shared_ptr& ps_ptr, - const std::shared_ptr& phis_ptr, - const std::shared_ptr& horiz_winds_ptr, - const std::shared_ptr& T_mid_ptr, - const std::shared_ptr& qv_ptr, - const std::shared_ptr& qc_ptr, - const std::shared_ptr& qr_ptr, - const std::shared_ptr& qi_ptr, - const std::shared_ptr& vapor_flux_ptr, - const std::shared_ptr& water_flux_ptr, - const std::shared_ptr& ice_flux_ptr, - const std::shared_ptr& heat_flux_ptr) + const Real mass_error_tolerance, + const Real energy_error_tolerance, + const Field& pseudo_density, + const Field& ps, + const Field& phis, + const Field& horiz_winds, + const Field& T_mid, + const Field& qv, + const Field& qc, + const Field& qr, + const Field& qi, + const Field& vapor_flux, + const Field& water_flux, + const Field& ice_flux, + const Field& heat_flux) : m_grid (grid) , m_dt (std::nan("")) , m_mass_tol (mass_error_tolerance) @@ -33,39 +35,19 @@ MassAndEnergyColumnConservationCheck (const std::shared_ptr& m_current_mass = view_1d ("current_total_water", m_num_cols); m_current_energy = view_1d ("current_total_energy", m_num_cols); - m_fields["pseudo_density"] = pseudo_density_ptr; - m_fields["ps"] = ps_ptr; - m_fields["phis"] = phis_ptr; - m_fields["horiz_winds"] = horiz_winds_ptr; - m_fields["T_mid"] = T_mid_ptr; - m_fields["qv"] = qv_ptr; - m_fields["qc"] = qc_ptr; - m_fields["qr"] = qr_ptr; - m_fields["qi"] = qi_ptr; - m_fields["vapor_flux"] = vapor_flux_ptr; - m_fields["water_flux"] = water_flux_ptr; - m_fields["ice_flux"] = ice_flux_ptr; - m_fields["heat_flux"] = heat_flux_ptr; - - // Require that all fields needed for mass and energy computation are not null. - const bool all_computation_fields_exist = - pseudo_density_ptr && ps_ptr && - phis_ptr && horiz_winds_ptr && - T_mid_ptr && qv_ptr && - qc_ptr && qr_ptr && - qi_ptr; - EKAT_REQUIRE_MSG(all_computation_fields_exist, - "Error! Currently we require mass and energy conservation " - "check to contain all fields related to the mass and energy " - "computation.\n"); - - // Require any process that add this checker to define all fluxes. - // Fluxes which are not relevant to a certain process should be set to 0. - EKAT_REQUIRE_MSG(vapor_flux_ptr && water_flux_ptr && - ice_flux_ptr && heat_flux_ptr, - "Error! If a process adds this check, it must define all " - "boundary fluxes. Fluxes which are not relevant to a " - "certain process should be set to 0.\n"); + m_fields["pseudo_density"] = pseudo_density; + m_fields["ps"] = ps; + m_fields["phis"] = phis; + m_fields["horiz_winds"] = horiz_winds; + m_fields["T_mid"] = T_mid; + m_fields["qv"] = qv; + m_fields["qc"] = qc; + m_fields["qr"] = qr; + m_fields["qi"] = qi; + m_fields["vapor_flux"] = vapor_flux; + m_fields["water_flux"] = water_flux; + m_fields["ice_flux"] = ice_flux; + m_fields["heat_flux"] = heat_flux; } void MassAndEnergyColumnConservationCheck::compute_current_mass () @@ -74,11 +56,11 @@ void MassAndEnergyColumnConservationCheck::compute_current_mass () const auto ncols = m_num_cols; const auto nlevs = m_num_levs; - const auto pseudo_density = m_fields.at("pseudo_density")->get_view(); - const auto qv = m_fields.at("qv")->get_view(); - const auto qc = m_fields.at("qc")->get_view(); - const auto qi = m_fields.at("qi")->get_view(); - const auto qr = m_fields.at("qr")->get_view(); + const auto pseudo_density = m_fields.at("pseudo_density").get_view(); + const auto qv = m_fields.at("qv").get_view(); + const auto qc = m_fields.at("qc").get_view(); + const auto qi = m_fields.at("qi").get_view(); + const auto qr = m_fields.at("qr").get_view(); const auto policy = ExeSpaceUtils::get_default_team_policy(ncols, nlevs); Kokkos::parallel_for(policy, KOKKOS_LAMBDA (const KT::MemberType& team) { @@ -100,14 +82,14 @@ void MassAndEnergyColumnConservationCheck::compute_current_energy () const auto ncols = m_num_cols; const auto nlevs = m_num_levs; - const auto pseudo_density = m_fields.at("pseudo_density")->get_view(); - const auto T_mid = m_fields.at("T_mid")->get_view(); - const auto horiz_winds = m_fields.at("horiz_winds")->get_view(); - const auto qv = m_fields.at("qv")->get_view(); - const auto qc = m_fields.at("qc")->get_view(); - const auto qr = m_fields.at("qr")->get_view(); - const auto ps = m_fields.at("ps")->get_view(); - const auto phis = m_fields.at("phis")->get_view(); + const auto pseudo_density = m_fields.at("pseudo_density").get_view(); + const auto T_mid = m_fields.at("T_mid").get_view(); + const auto horiz_winds = m_fields.at("horiz_winds").get_view(); + const auto qv = m_fields.at("qv").get_view(); + const auto qc = m_fields.at("qc").get_view(); + const auto qr = m_fields.at("qr").get_view(); + const auto ps = m_fields.at("ps").get_view(); + const auto phis = m_fields.at("phis").get_view(); const auto policy = ExeSpaceUtils::get_default_team_policy(ncols, nlevs); Kokkos::parallel_for(policy, KOKKOS_LAMBDA (const KT::MemberType& team) { @@ -136,20 +118,20 @@ PropertyCheck::ResultAndMsg MassAndEnergyColumnConservationCheck::check() const "before running check()."); auto dt = m_dt; - const auto pseudo_density = m_fields.at("pseudo_density")->get_view (); - const auto T_mid = m_fields.at("T_mid" )->get_view (); - const auto horiz_winds = m_fields.at("horiz_winds" )->get_view(); - const auto qv = m_fields.at("qv" )->get_view (); - const auto qc = m_fields.at("qc" )->get_view (); - const auto qi = m_fields.at("qi" )->get_view (); - const auto qr = m_fields.at("qr" )->get_view (); - const auto ps = m_fields.at("ps" )->get_view (); - const auto phis = m_fields.at("phis" )->get_view (); - - const auto vapor_flux = m_fields.at("vapor_flux")->get_view(); - const auto water_flux = m_fields.at("water_flux")->get_view(); - const auto ice_flux = m_fields.at("ice_flux" )->get_view(); - const auto heat_flux = m_fields.at("heat_flux" )->get_view(); + const auto pseudo_density = m_fields.at("pseudo_density").get_view (); + const auto T_mid = m_fields.at("T_mid" ).get_view (); + const auto horiz_winds = m_fields.at("horiz_winds" ).get_view(); + const auto qv = m_fields.at("qv" ).get_view (); + const auto qc = m_fields.at("qc" ).get_view (); + const auto qi = m_fields.at("qi" ).get_view (); + const auto qr = m_fields.at("qr" ).get_view (); + const auto ps = m_fields.at("ps" ).get_view (); + const auto phis = m_fields.at("phis" ).get_view (); + + const auto vapor_flux = m_fields.at("vapor_flux").get_view(); + const auto water_flux = m_fields.at("water_flux").get_view(); + const auto ice_flux = m_fields.at("ice_flux" ).get_view(); + const auto heat_flux = m_fields.at("heat_flux" ).get_view(); // Use Kokkos::MaxLoc to find the largest error for both mass and energy using maxloc_t = Kokkos::MaxLoc; @@ -237,14 +219,16 @@ PropertyCheck::ResultAndMsg MassAndEnergyColumnConservationCheck::check() const res_and_msg.result = CheckResult::Fail; // We output relative errors with lat/lon information (if available) - using gid_t = AbstractGrid::gid_type; - auto gids = m_grid->get_dofs_gids().get_view(); + using gid_type = AbstractGrid::gid_type; + auto gids = m_grid->get_dofs_gids().get_view(); typename Field::view_host_t lat, lon; const bool has_latlon = m_grid->has_geometry_data("lat") && m_grid->has_geometry_data("lon"); if (has_latlon) { lat = m_grid->get_geometry_data("lat").get_view(); lon = m_grid->get_geometry_data("lon").get_view(); } + const bool has_additional_col_info = not additional_data_fields().empty(); + using namespace ShortFieldTagsNames; std::stringstream msg; msg << "Check failed.\n" @@ -256,8 +240,17 @@ PropertyCheck::ResultAndMsg MassAndEnergyColumnConservationCheck::check() const if (has_latlon) { msg << " - (lat, lon): (" << lat(maxloc_mass.loc) << ", " << lon(maxloc_mass.loc) << ")\n"; } + if (has_additional_col_info) { + msg << " - additional data (w/ local column index):\n"; + for (auto& f : additional_data_fields()) { + f.sync_to_host(); + msg << "\n"; + print_field_hyperslab(f, {COL}, {maxloc_mass.loc}, msg); + } + msg << "\n END OF ADDITIONAL DATA\n"; + } res_and_msg.fail_loc_indices.resize(1,maxloc_mass.loc); - res_and_msg.fail_loc_tags = m_fields.at("phis")->get_header().get_identifier().get_layout().tags(); + res_and_msg.fail_loc_tags = m_fields.at("phis").get_header().get_identifier().get_layout().tags(); } if (not energy_below_tol) { msg << " - energy error tolerance: " << m_energy_tol << "\n"; @@ -266,8 +259,17 @@ PropertyCheck::ResultAndMsg MassAndEnergyColumnConservationCheck::check() const if (has_latlon) { msg << " - (lat, lon): (" << lat(maxloc_energy.loc) << ", " << lon(maxloc_energy.loc) << ")\n"; } + if (has_additional_col_info) { + msg << " - additional data (w/ local column index):\n"; + for (auto& f : additional_data_fields()) { + f.sync_to_host(); + msg << "\n"; + print_field_hyperslab(f, {COL}, {maxloc_energy.loc}, msg); + } + msg << "\n END OF ADDITIONAL DATA\n"; + } res_and_msg.fail_loc_indices.resize(1,maxloc_energy.loc); - res_and_msg.fail_loc_tags = m_fields.at("phis")->get_header().get_identifier().get_layout().tags(); + res_and_msg.fail_loc_tags = m_fields.at("phis").get_header().get_identifier().get_layout().tags(); } res_and_msg.msg = msg.str(); diff --git a/components/eamxx/src/share/property_checks/mass_and_energy_column_conservation_check.hpp b/components/eamxx/src/share/property_checks/mass_and_energy_column_conservation_check.hpp index 2b99eea419c2..cd15cb4e0aa4 100644 --- a/components/eamxx/src/share/property_checks/mass_and_energy_column_conservation_check.hpp +++ b/components/eamxx/src/share/property_checks/mass_and_energy_column_conservation_check.hpp @@ -30,21 +30,21 @@ class MassAndEnergyColumnConservationCheck: public PropertyCheck { // Constructor MassAndEnergyColumnConservationCheck (const std::shared_ptr& grid, - const Real mass_error_tolerance, - const Real energy_error_tolerance, - const std::shared_ptr& pseudo_density_ptr, - const std::shared_ptr& ps_ptr, - const std::shared_ptr& phis_ptr, - const std::shared_ptr& horiz_winds_ptr, - const std::shared_ptr& T_mid_ptr, - const std::shared_ptr& qv_ptr, - const std::shared_ptr& qc_ptr, - const std::shared_ptr& qr_ptr, - const std::shared_ptr& qi_ptr, - const std::shared_ptr& vapor_flux_ptr, - const std::shared_ptr& water_flux_ptr, - const std::shared_ptr& ice_flux_ptr, - const std::shared_ptr& heat_flux_ptr); + const Real mass_error_tolerance, + const Real energy_error_tolerance, + const Field& pseudo_density_ptr, + const Field& ps_ptr, + const Field& phis_ptr, + const Field& horiz_winds_ptr, + const Field& T_mid_ptr, + const Field& qv_ptr, + const Field& qc_ptr, + const Field& qr_ptr, + const Field& qi_ptr, + const Field& vapor_flux_ptr, + const Field& water_flux_ptr, + const Field& ice_flux_ptr, + const Field& heat_flux_ptr); // The name of the property check std::string name () const override { return "Mass and energy column conservation check"; } @@ -111,8 +111,8 @@ class MassAndEnergyColumnConservationCheck: public PropertyCheck { protected: - std::shared_ptr m_grid; - std::map> m_fields; + std::shared_ptr m_grid; + std::map m_fields; int m_num_cols; int m_num_levs; diff --git a/components/eamxx/src/share/property_checks/property_check.cpp b/components/eamxx/src/share/property_checks/property_check.cpp index 0e9f4dd7d95a..073a303ec66a 100644 --- a/components/eamxx/src/share/property_checks/property_check.cpp +++ b/components/eamxx/src/share/property_checks/property_check.cpp @@ -52,6 +52,22 @@ set_fields (const std::list& fields, } } +void PropertyCheck:: +set_additional_data_field (const Field& data_field) +{ + EKAT_REQUIRE_MSG(data_field.get_header().get_identifier().get_layout().has_tag(FieldTag::Column), + "Error! Additional data field \""+data_field.name()+"\" for property check \"" + +name()+"\" must be defined on columns.\n"); + + // Only add field if it currently does not exist in additional fields list. + const bool found_field_in_list = std::find(m_additional_data_fields.begin(), + m_additional_data_fields.end(), + data_field) != m_additional_data_fields.end(); + if (not found_field_in_list) { + m_additional_data_fields.push_back(data_field); + } +} + // If a check fails, attempt to repair things. Default is to throw. void PropertyCheck::repair () const { EKAT_REQUIRE_MSG (can_repair(), diff --git a/components/eamxx/src/share/property_checks/property_check.hpp b/components/eamxx/src/share/property_checks/property_check.hpp index c4d6c49ea874..1e891ff787ad 100644 --- a/components/eamxx/src/share/property_checks/property_check.hpp +++ b/components/eamxx/src/share/property_checks/property_check.hpp @@ -13,7 +13,7 @@ namespace scream /* * Abstract interface for property checks - * + * * A property check (PC) object is responsible to check that * a certain property holds. The class can (but does not have to) * also implement a way to 'repair' the simulation if @@ -21,13 +21,13 @@ namespace scream * * PC's are stored in an AtmosphereProcess (AP), and can be * run before and/or after the process is executed. - * + * * The typical class deriving from this interface will implement * some checks on one or more Field objects, verifying that they * satisfy the properties. For instance, we can check that a * Field does not contain NaN values, or that it is always within * certain bounds. - * + * * More complicate checks can verify that 2+ fields together verify * a certain property. For instance, we might want to verify that * the sum of certain quantities stay the same (like an energy or @@ -76,6 +76,10 @@ class PropertyCheck { void set_fields (const std::list& fields, const std::list& repairable); + // Additional column data fields can be added to output + // stream of any property check. + void set_additional_data_field (const Field& data_field); + // Whether this PC is capable of fixing things if the check fails. // Defaults to false. bool can_repair() const { @@ -97,6 +101,11 @@ class PropertyCheck { return m_repairable_fields; } + // Return additional data fields used in this property check + const std::list& additional_data_fields () const { + return m_additional_data_fields; + } + // If a check fails, attempt to repair things. Default is to throw. void repair () const; @@ -117,6 +126,8 @@ class PropertyCheck { std::list m_fields; std::list m_repairable_fields; + + std::list m_additional_data_fields; }; } // namespace scream diff --git a/components/eamxx/src/share/scream_config.cpp b/components/eamxx/src/share/scream_config.cpp index c1603cbd9042..88a9ca48071f 100644 --- a/components/eamxx/src/share/scream_config.cpp +++ b/components/eamxx/src/share/scream_config.cpp @@ -21,4 +21,21 @@ std::string scream_config_string() { return config; } +bool& use_leap_year_impl () { +#ifdef SCREAM_HAS_LEAP_YEAR + static bool use_leap = true; +#else + static bool use_leap = false; +#endif + return use_leap; +} + +bool use_leap_year () { + return use_leap_year_impl (); +} + +void set_use_leap_year (const bool use_leap) { + use_leap_year_impl () = use_leap; +} + } // namespace scream diff --git a/components/eamxx/src/share/scream_config.hpp b/components/eamxx/src/share/scream_config.hpp index fd0a35a4be50..fea495c5ae3f 100644 --- a/components/eamxx/src/share/scream_config.hpp +++ b/components/eamxx/src/share/scream_config.hpp @@ -18,6 +18,10 @@ namespace scream { std::string scream_config_string(); +// Utils to set/get whether leap year is used or not +bool use_leap_year (); +void set_use_leap_year (const bool use_leap); + } // namespace scream #endif // SCREAM_CONFIG_HPP diff --git a/components/eamxx/src/share/tests/CMakeLists.txt b/components/eamxx/src/share/tests/CMakeLists.txt index 1a5555df100c..0b2363fad2f6 100644 --- a/components/eamxx/src/share/tests/CMakeLists.txt +++ b/components/eamxx/src/share/tests/CMakeLists.txt @@ -3,49 +3,71 @@ if (NOT ${SCREAM_BASELINES_ONLY}) include(ScreamUtils) # Test vertical interpolation - CreateUnitTest(vertical_interp "vertical_interp_tests.cpp" scream_share) + CreateUnitTest(vertical_interp "vertical_interp_tests.cpp") # Test utils - CreateUnitTest(utils "utils_tests.cpp" scream_share) + CreateUnitTest(utils "utils_tests.cpp") # Test column ops - CreateUnitTest(column_ops "column_ops.cpp" scream_share) + CreateUnitTest(column_ops "column_ops.cpp") # Test fields - CreateUnitTest(field "field_tests.cpp" scream_share) + CreateUnitTest(field "field_tests.cpp") # Test field utils - CreateUnitTest(field_utils "field_utils.cpp" scream_share + CreateUnitTest(field_utils "field_utils.cpp" MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) # Test property checks - CreateUnitTest(property_checks "property_checks.cpp" scream_share) + CreateUnitTest(property_checks "property_checks.cpp") # Test grids - CreateUnitTest(grid "grid_tests.cpp" scream_share + CreateUnitTest(grid "grid_tests.cpp" MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) - # Test coarsening remap - CreateUnitTest(coarsening_remapper "coarsening_remapper_tests.cpp" "scream_share;scream_io" + # Test grid import-export + CreateUnitTest(grid_imp_exp "grid_import_export_tests.cpp" MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) # Test coarsening remap - CreateUnitTest(vertical_remapper "vertical_remapper_tests.cpp" "scream_share;scream_io" + CreateUnitTest(coarsening_remapper "coarsening_remapper_tests.cpp" + LIBS scream_io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) + + if (EAMXX_ENABLE_EXPERIMENTAL_CODE) + # Test refining remap (RMA version) + CreateUnitTest(refining_remapper_rma "refining_remapper_rma_tests.cpp" + LIBS scream_io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) + endif() + + # Test refining remap (P2P version) + CreateUnitTest(refining_remapper_p2p "refining_remapper_p2p_tests.cpp" + LIBS scream_io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) + + # Test vertical remap + CreateUnitTest(vertical_remapper "vertical_remapper_tests.cpp" + LIBS scream_io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) + + # Test vertical remap + CreateUnitTest(time_interpolation "eamxx_time_interpolation_tests.cpp" + LIBS scream_io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) # Test common physics functions - CreateUnitTest(common_physics "common_physics_functions_tests.cpp" scream_share) + CreateUnitTest(common_physics "common_physics_functions_tests.cpp") # Test atmosphere processes - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/atm_process_tests_parse_list.yaml - ${CMAKE_CURRENT_BINARY_DIR}/atm_process_tests_parse_list.yaml COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/atm_process_tests_named_procs.yaml ${CMAKE_CURRENT_BINARY_DIR}/atm_process_tests_named_procs.yaml COPYONLY) - CreateUnitTest(atm_proc "atm_process_tests.cpp" scream_share) + CreateUnitTest(atm_proc "atm_process_tests.cpp") # Test horizontal remapping utility - CreateUnitTest(horizontal_remap "horizontal_remap_test.cpp" "scream_share;scream_io" - LABELS "horiz_remap" + CreateUnitTest(horizontal_remap "horizontal_remap_test.cpp" + LIBS scream_io + LABELS horiz_remap MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) endif() diff --git a/components/eamxx/src/share/tests/atm_process_tests.cpp b/components/eamxx/src/share/tests/atm_process_tests.cpp index 6fbf60ad55b5..01985c33c6d0 100644 --- a/components/eamxx/src/share/tests/atm_process_tests.cpp +++ b/components/eamxx/src/share/tests/atm_process_tests.cpp @@ -22,18 +22,20 @@ namespace scream { ekat::ParameterList create_test_params () { + using strvec_t = std::vector; + // Create a parameter list for inputs ekat::ParameterList params ("Atmosphere Processes"); params.set("schedule_type","Sequential"); - params.set("atm_procs_list","(Foo,BarBaz)"); + params.set("atm_procs_list",{"Foo","BarBaz"}); auto& p0 = params.sublist("Foo"); p0.set("Type", "Foo"); p0.set("Grid Name", "Point Grid"); auto& p1 = params.sublist("BarBaz"); - p1.set("atm_procs_list","(Bar,Baz)"); + p1.set("atm_procs_list",{"Bar","Baz"}); p1.set("Type", "Group"); p1.set("schedule_type","Sequential"); @@ -57,13 +59,7 @@ create_gm (const ekat::Comm& comm) { const int num_local_cols = 13; const int num_global_cols = num_local_cols*comm.size(); - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns", num_global_cols); - gm_params.set("number_of_local_elements", num_local_elems); - gm_params.set("number_of_vertical_levels", nlevs); - gm_params.set("number_of_gauss_points", np); - - auto gm = create_mesh_free_grids_manager(comm,gm_params); + auto gm = create_mesh_free_grids_manager(comm,num_local_elems,np,nlevs,num_global_cols); gm->build_grids(); return gm; @@ -354,59 +350,31 @@ TEST_CASE("process_factory", "") { factory.register_product("grouP",&create_atmosphere_process); factory.register_product("DiagIdentity",&create_atmosphere_process); - // Create the processes - SECTION ("parse_list") { - // Load ad parameter list - std::string fname = "atm_process_tests_parse_list.yaml"; - ekat::ParameterList params ("Atmosphere Processes"); - REQUIRE_NOTHROW ( parse_yaml_file(fname,params) ); - - std::shared_ptr atm_process (factory.create("group",comm,params)); - - // CHECKS - auto group = std::dynamic_pointer_cast(atm_process); - - // 1) Must be a group - REQUIRE (static_cast(group)); - - // 2) Must store 2 processes: a Physics and a Group - REQUIRE (group->get_num_processes()==2); - REQUIRE (group->get_process(0)->type()==AtmosphereProcessType::Dynamics); - REQUIRE (group->get_process(1)->type()==AtmosphereProcessType::Group); - - // 3) The group must store two physics - auto group_2 = std::dynamic_pointer_cast(group->get_process(1)); - REQUIRE (static_cast(group_2)); - REQUIRE (group_2->get_num_processes()==2); - REQUIRE (group_2->get_process(0)->type()==AtmosphereProcessType::Physics); - REQUIRE (group_2->get_process(1)->type()==AtmosphereProcessType::Physics); - } + // Load ad parameter list + std::string fname = "atm_process_tests_named_procs.yaml"; + ekat::ParameterList params ("Atmosphere Processes"); + parse_yaml_file(fname,params); - SECTION ("named_procs") { - // Load ad parameter list - std::string fname = "atm_process_tests_named_procs.yaml"; - ekat::ParameterList params ("Atmosphere Processes"); - REQUIRE_NOTHROW ( parse_yaml_file(fname,params) ); - std::shared_ptr atm_process (factory.create("group",comm,params)); + // Create the processes + std::shared_ptr atm_process (factory.create("group",comm,params)); - // CHECKS - auto group = std::dynamic_pointer_cast(atm_process); + // CHECKS + auto group = std::dynamic_pointer_cast(atm_process); - // 1) Must be a group - REQUIRE (static_cast(group)); + // 1) Must be a group + REQUIRE (static_cast(group)); - // 2) Must store 2 processes: a Physics and a Group - REQUIRE (group->get_num_processes()==2); - REQUIRE (group->get_process(0)->type()==AtmosphereProcessType::Dynamics); - REQUIRE (group->get_process(1)->type()==AtmosphereProcessType::Group); + // 2) Must store 2 processes: a Physics and a Group + REQUIRE (group->get_num_processes()==2); + REQUIRE (group->get_process(0)->type()==AtmosphereProcessType::Dynamics); + REQUIRE (group->get_process(1)->type()==AtmosphereProcessType::Group); - // 3) The group must store two physics - auto group_2 = std::dynamic_pointer_cast(group->get_process(1)); - REQUIRE (static_cast(group_2)); - REQUIRE (group_2->get_num_processes()==2); - REQUIRE (group_2->get_process(0)->type()==AtmosphereProcessType::Physics); - REQUIRE (group_2->get_process(1)->type()==AtmosphereProcessType::Physics); - } + // 3) The group must store two physics + auto group_2 = std::dynamic_pointer_cast(group->get_process(1)); + REQUIRE (static_cast(group_2)); + REQUIRE (group_2->get_num_processes()==2); + REQUIRE (group_2->get_process(0)->type()==AtmosphereProcessType::Physics); + REQUIRE (group_2->get_process(1)->type()==AtmosphereProcessType::Physics); } TEST_CASE("atm_proc_dag", "") { @@ -473,11 +441,12 @@ TEST_CASE("atm_proc_dag", "") { SECTION ("broken") { + using strvec_t = std::vector; auto params = create_test_params(); auto p1 = params.sublist("BarBaz"); // Make sure there's a missing piece (whatever Baz computes); - p1.set("atm_procs_list","(Bar)"); + p1.set("atm_procs_list",{"Bar"}); std::shared_ptr broken_atm_group (factory.create("group",comm,params)); broken_atm_group->set_grids(gm); @@ -546,7 +515,7 @@ TEST_CASE("field_checks", "") { if (not allow_failure && (check_pre || check_post)) { REQUIRE_THROWS (foo->run(1)); } else { - REQUIRE_NOTHROW (foo->run(1)); + foo->run(1); } } } diff --git a/components/eamxx/src/share/tests/atm_process_tests_named_procs.yaml b/components/eamxx/src/share/tests/atm_process_tests_named_procs.yaml index 05610209a829..16eda725b137 100644 --- a/components/eamxx/src/share/tests/atm_process_tests_named_procs.yaml +++ b/components/eamxx/src/share/tests/atm_process_tests_named_procs.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -atm_procs_list: (MyFoo,BarBaz) +atm_procs_list: [MyFoo,BarBaz] schedule_type: Sequential Type: Group @@ -10,7 +10,7 @@ MyFoo: BarBaz: Type: Group schedule_type: Sequential - atm_procs_list: (MyBar,MyBaz) + atm_procs_list: [MyBar,MyBaz] MyBar: Type: Bar diff --git a/components/eamxx/src/share/tests/atm_process_tests_parse_list.yaml b/components/eamxx/src/share/tests/atm_process_tests_parse_list.yaml deleted file mode 100644 index 4649e26480a0..000000000000 --- a/components/eamxx/src/share/tests/atm_process_tests_parse_list.yaml +++ /dev/null @@ -1,20 +0,0 @@ -%YAML 1.1 ---- -atm_procs_list: (MyFoo,(MyBar,MyBaz)) -schedule_type: Sequential -Type: Group - -MyFoo: - Type: Foo - Grid Name: Point Grid -group.MyBar_MyBaz.: - schedule_type: Sequential - atm_procs_list: (MyBar,MyBaz) - - MyBar: - Type: Bar - Grid Name: Point Grid - MyBaz: - Type: Baz - Grid Name: Point Grid -... diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index f74aaa3f96c3..e56b43aab7ef 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -3,28 +3,27 @@ #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/point_grid.hpp" #include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/field/field_utils.hpp" namespace scream { -template -typename ViewT::HostMirror -cmvc (const ViewT& v) { - auto vh = Kokkos::create_mirror_view(v); - Kokkos::deep_copy(vh,v); - return vh; -} - class CoarseningRemapperTester : public CoarseningRemapper { public: + using gid_type = AbstractGrid::gid_type; + CoarseningRemapperTester (const grid_ptr_type& src_grid, const std::string& map_file) : CoarseningRemapper(src_grid,map_file) { // Nothing to do } - std::vector - test_triplet_gids (const std::string& map_file) const { - return CoarseningRemapper::get_my_triplets_gids (map_file,m_src_grid); + + // Note: we use this instead of get_tgt_grid, b/c the nonconst grid + // will give use a not read-only gids field, so we can pass + // pointers to MPI_Bcast (which needs pointer to nonconst) + std::shared_ptr get_coarse_grid () const { + return m_coarse_grid; } view_1d get_row_offsets () const { @@ -38,116 +37,201 @@ class CoarseningRemapperTester : public CoarseningRemapper { } grid_ptr_type get_ov_tgt_grid () const { - return m_ov_tgt_grid; + return m_ov_coarse_grid; } view_2d::HostMirror get_send_f_pid_offsets () const { - return cmvc(m_send_f_pid_offsets); + return cmvdc(m_send_f_pid_offsets); } view_2d::HostMirror get_recv_f_pid_offsets () const { - return cmvc(m_recv_f_pid_offsets); + return cmvdc(m_recv_f_pid_offsets); } view_1d::HostMirror get_recv_lids_beg () const { - return cmvc(m_recv_lids_beg); + return cmvdc(m_recv_lids_beg); } view_1d::HostMirror get_recv_lids_end () const { - return cmvc(m_recv_lids_end); + return cmvdc(m_recv_lids_end); } view_2d::HostMirror get_send_lids_pids () const { - return cmvc(m_send_lids_pids ); + return cmvdc(m_send_lids_pids ); } view_2d::HostMirror get_recv_lids_pidpos () const { - return cmvc(m_recv_lids_pidpos); + return cmvdc(m_recv_lids_pidpos); } view_1d::HostMirror get_send_pid_lids_start () const { - return cmvc(m_send_pid_lids_start); - } - - int gid2lid (const gid_t gid, const grid_ptr_type& grid) const { - return CoarseningRemapper::gid2lid(gid,grid); + return cmvdc(m_send_pid_lids_start); } }; -template -bool view_contains (const ViewT& v, const typename ViewT::traits::value_type& entry) { - const auto vh = cmvc (v); - const auto beg = vh.data(); - const auto end = vh.data() + vh.size(); - for (auto it=beg; it!=end; ++it) { - if (*it == entry) { - return true; - } - } - return false; -} - -void print (const std::string& msg, const ekat::Comm& comm) { +void root_print (const std::string& msg, const ekat::Comm& comm) { if (comm.am_i_root()) { printf("%s",msg.c_str()); } } -// Helper function to create a grid given the number of dof's and a comm group. +// Create a source grid given number of global dofs. +// Dofs are scattered around randomly +template std::shared_ptr -build_src_grid(const ekat::Comm& comm, const int nldofs_src) +build_src_grid(const ekat::Comm& comm, const int ngdofs, Engine& engine) { - auto src_grid = std::make_shared("src",nldofs_src,20,comm); + using gid_type = AbstractGrid::gid_type; + const int nlevs = 20; + + std::vector all_dofs (ngdofs); + if (comm.am_i_root()) { + std::iota(all_dofs.data(),all_dofs.data()+all_dofs.size(),0); + std::shuffle(all_dofs.data(),all_dofs.data()+ngdofs,engine); + } + comm.broadcast(all_dofs.data(),ngdofs,comm.root_rank()); + + int nldofs = ngdofs / comm.size(); + int remainder = ngdofs % comm.size(); + int offset = nldofs * comm.rank() + std::min(comm.rank(),remainder); + if (comm.rank()("src",nldofs,nlevs,comm); auto src_dofs = src_grid->get_dofs_gids(); - auto src_dofs_h = src_dofs.get_view(); - std::iota(src_dofs_h.data(),src_dofs_h.data()+nldofs_src,nldofs_src*comm.rank()); + auto src_dofs_h = src_dofs.get_view(); + std::copy_n(all_dofs.data()+offset,nldofs,src_dofs_h.data()); src_dofs.sync_to_dev(); return src_grid; } -// Helper function to create fields -Field -create_field(const std::string& name, const std::shared_ptr& grid, const bool twod, const bool vec, const bool mid = false, const int ps = 1, const bool add_mask = false) +constexpr int vec_dim = 2; +Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid, const bool midpoints) { - constexpr int vec_dim = 3; - constexpr auto CMP = FieldTag::Component; - constexpr auto units = ekat::units::Units::nondimensional(); - auto fl = twod - ? (vec ? grid->get_2d_vector_layout (CMP,vec_dim) - : grid->get_2d_scalar_layout ()) - : (vec ? grid->get_3d_vector_layout (mid,CMP,vec_dim) - : grid->get_3d_scalar_layout (mid)); - FieldIdentifier fid(name,fl,units,grid->name()); - Field f(fid); - f.get_header().get_alloc_properties().request_allocation(ps); + const auto u = ekat::units::Units::nondimensional(); + const auto CMP = ShortFieldTagsNames::CMP; + const auto& gn = grid.name(); + Field f; + switch (lt) { + case LayoutType::Scalar2D: + f = Field(FieldIdentifier(name,grid.get_2d_scalar_layout(),u,gn)); break; + case LayoutType::Vector2D: + f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(CMP,vec_dim),u,gn)); break; + case LayoutType::Scalar3D: + f = Field(FieldIdentifier(name,grid.get_3d_scalar_layout(midpoints),u,gn)); + f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + break; + case LayoutType::Vector3D: + f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(midpoints,CMP,vec_dim),u,gn)); + f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + break; + default: + EKAT_ERROR_MSG ("Invalid layout type for this unit test.\n"); + } f.allocate_view(); - if (add_mask) { - // Add a mask to the field - FieldIdentifier fid_mask(name+"_mask",fl,units,grid->name()); - Field f_mask(fid_mask); - f_mask.get_header().get_alloc_properties().request_allocation(ps); - f_mask.allocate_view(); - f.get_header().set_extra_data("mask_data",f_mask); - } + return f; +} + +template +Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid, const bool midpoints, Engine& engine) { + auto f = create_field(name,lt,grid,midpoints); + + // Use discrete_distribution to get an integer, then use that as exponent for 2^-n. + // This guarantees numbers that are exactly represented as FP numbers, which ensures + // the test will produce the expected answer, regardless of how math ops are performed. + using IPDF = std::discrete_distribution; + IPDF ipdf ({1,1,1,1,1,1,1,1,1,1}); + auto pdf = [&](Engine& e) { + return Real(std::pow(2,ipdf(e))); + }; + randomize(f,engine,pdf); return f; } +template +Field all_gather_field_impl (const Field& f, const ekat::Comm& comm) { + constexpr auto COL = ShortFieldTagsNames::COL; + const auto& fid = f.get_header().get_identifier(); + const auto& fl = fid.get_layout(); + int col_size = fl.strip_dim(COL).size(); + auto tags = fl.tags(); + auto dims = fl.dims(); + int my_cols = dims[0];; + comm.all_reduce(&my_cols, &dims.front(), 1, MPI_SUM ); + FieldLayout gfl(tags,dims); + FieldIdentifier gfid("g" + f.name(),gfl,fid.get_units(),fid.get_grid_name(),fid.data_type()); + Field gf(gfid); + gf.allocate_view(); + std::vector data_vec(col_size); + f.sync_to_host(); + for (int pid=0,offset=0; pid(),icol).data(); + } else { + data = data_vec.data(); + } + break; + case 2: + if (pid==comm.rank()) { + data = ekat::subview(f.get_view(),icol).data(); + } else { + data = data_vec.data(); + } + break; + case 3: + if (pid==comm.rank()) { + data = ekat::subview(f.get_view(),icol).data(); + } else { + data = data_vec.data(); + } + break; + default: + EKAT_ERROR_MSG ( + "Unexpected rank in RefiningRemapperRMA unit test.\n" + " - field name: " + f.name() + "\n"); + } + comm.broadcast(data,col_size,pid); + auto gdata = gf.get_internal_view_data()+offset; + std::copy(data,data+col_size,gdata); + } + } + return gf; +} + +Field all_gather_field (const Field& f, const ekat::Comm& comm) { + const auto dt = f.data_type(); + if (dt==DataType::RealType) { + return all_gather_field_impl(f,comm); + } else { + return all_gather_field_impl(f,comm); + } +} + // Helper function to create a remap file -void create_remap_file(const std::string& filename, std::vector& dofs, - const int na, const int nb, const int ns, - const std::vector& col, const std::vector& row, const std::vector& S) +void create_remap_file(const std::string& filename, const int ngdofs_tgt) { + const int ngdofs_src = ngdofs_tgt + 1; + const int nnz = 2*ngdofs_tgt; scorpio::register_file(filename, scorpio::FileMode::Write); - scorpio::register_dimension(filename,"n_a", "n_a", na, true); - scorpio::register_dimension(filename,"n_b", "n_b", nb, true); - scorpio::register_dimension(filename,"n_s", "n_s", ns, true); + scorpio::register_dimension(filename,"n_a", "n_a", ngdofs_src, true); + scorpio::register_dimension(filename,"n_b", "n_b", ngdofs_tgt, true); + scorpio::register_dimension(filename,"n_s", "n_s", nnz, true); - scorpio::register_variable(filename,"col","col","none",{"n_s"},"real","int","int-nnz"); - scorpio::register_variable(filename,"row","row","none",{"n_s"},"real","int","int-nnz"); - scorpio::register_variable(filename,"S","S","none",{"n_s"},"real","real","Real-nnz"); + scorpio::register_variable(filename,"col","col","none",{"n_s"},"int","int","int-nnz"); + scorpio::register_variable(filename,"row","row","none",{"n_s"},"int","int","int-nnz"); + scorpio::register_variable(filename,"S","S","none",{"n_s"},"double","double","Real-nnz"); + + std::vector dofs(nnz); + std::iota(dofs.begin(),dofs.end(),0); scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); @@ -155,14 +239,26 @@ void create_remap_file(const std::string& filename, std::vector& d scorpio::eam_pio_enddef(filename); - scorpio::grid_write_data_array(filename,"row",row.data(),ns); - scorpio::grid_write_data_array(filename,"col",col.data(),ns); - scorpio::grid_write_data_array(filename,"S", S.data(),ns); + std::vector col(nnz), row(nnz); + std::vector S(nnz,0.5); + for (int i=0; insrc") { +TEST_CASE("coarsening_remap") +{ + auto& catch_capture = Catch::getResultCapture(); + // This is a simple test to just make sure the coarsening remapper works // when the map itself has more remap triplets than the size of the // source and target grid. This is typical in monotone remappers from @@ -174,507 +270,177 @@ TEST_CASE("coarsening_remap_nnz>nsrc") { ekat::Comm comm(MPI_COMM_WORLD); + root_print ("\n +---------------------------------+\n",comm); + root_print (" | Testing coarsening remapper |\n",comm); + root_print (" +---------------------------------+\n\n",comm); + MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); scorpio::eam_init_pio_subsystem(fcomm); - - // -------------------------------------- // - // Set grid/map sizes // - // -------------------------------------- // - - const int nldofs_src = 4; - const int nldofs_tgt = 2; - const int ngdofs_src = nldofs_src*comm.size(); - const int ngdofs_tgt = nldofs_tgt*comm.size(); - const int nnz_local = nldofs_src*nldofs_tgt; - const int nnz = nnz_local*comm.size(); + auto engine = setup_random_test (&comm); // -------------------------------------- // // Create a map file // // -------------------------------------- // - print (" -> creating map file ...\n",comm); - - std::string filename = "coarsening_map_file_lrg_np" + std::to_string(comm.size()) + ".nc"; - std::vector dofs (nnz_local); - std::iota(dofs.begin(),dofs.end(),comm.rank()*nnz_local); - - // Create triplets: tgt entry K is the avg of src entries K and K+ngdofs_tgt - // NOTE: add 1 to row/col indices, since e3sm map files indices are 1-based - std::vector col,row,S; - const Real wgt = 1.0/nldofs_src; - for (int i=0; i creating map file ... done!\n",comm); + const int nldofs_tgt = 2; + const int ngdofs_tgt = nldofs_tgt*comm.size(); + create_remap_file(filename, ngdofs_tgt); // -------------------------------------- // // Build src grid and remapper // // -------------------------------------- // - print (" -> creating grid ...\n",comm); - auto src_grid = build_src_grid(comm, nldofs_src); - print (" -> creating grid ... done\n",comm); - - print (" -> creating remapper ...\n",comm); + const int ngdofs_src = ngdofs_tgt+1; + auto src_grid = build_src_grid(comm, ngdofs_src, engine); auto remap = std::make_shared(src_grid,filename); - print (" -> creating remapper ... done!\n",comm); // -------------------------------------- // // Create src/tgt grid fields // // -------------------------------------- // - print (" -> creating fields ...\n",comm); // The other test checks remapping for fields of multiple dimensions. // Here we will simplify and just remap a simple 2D horizontal field. - auto tgt_grid = remap->get_tgt_grid(); - - auto src_s2d = create_field("s2d", src_grid,true,false,false,1,true); - auto tgt_s2d = create_field("s2d", tgt_grid,true,false); - - std::vector src_f = {src_s2d}; - std::vector tgt_f = {tgt_s2d}; - - // -------------------------------------- // - // Register fields in the remapper // - // -------------------------------------- // - - print (" -> registering fields ...\n",comm); - remap->registration_begins(); - remap->register_field(src_s2d, tgt_s2d); - remap->registration_ends(); - print (" -> registering fields ... done!\n",comm); - - // -------------------------------------- // - // Generate data for src fields // - // -------------------------------------- // - - print (" -> generate src fields data ...\n",comm); - // Generate data in a deterministic way, so that when we check results, - // we know a priori what the input data that generated the tgt field's - // values was, even if that data was off rank. - auto src_gids = remap->get_src_grid()->get_dofs_gids().get_view(); - for (const auto& f : src_f) { - const auto& l = f.get_header().get_identifier().get_layout(); - switch (get_layout_type(l.tags())) { - case LayoutType::Scalar2D: - { - const auto v_src = f.get_view(); - for (int i=0; i generate src fields data ... done!\n",comm); - - - // -------------------------------------- // - // Check remapped fields // - // -------------------------------------- // - const auto tgt_gids = tgt_grid->get_dofs_gids().get_view(); - for (int irun=0; irun<5; ++irun) { - print (" -> run remap ...\n",comm); - remap->remap(true); - print (" -> run remap ... done!\n",comm); - - print (" -> check tgt fields ...\n",comm); - // Recall, tgt gid K should be the avg of local src_gids - const int ntgt_gids = tgt_gids.size(); - for (size_t ifield=0; ifield Checking field with layout " + to_string(l) + " " + dots + "\n",comm); - - f.sync_to_host(); - - switch (get_layout_type(l.tags())) { - case LayoutType::Scalar2D: - { - const auto v_tgt = f.get_view(); - for (int i=0; i creating map file ...\n",comm); - - std::string filename = "coarsening_map_file_np" + std::to_string(comm.size()) + ".nc"; - std::vector dofs (nnz_local); - std::iota(dofs.begin(),dofs.end(),comm.rank()*nnz_local); - - // Create triplets: tgt entry K is the avg of src entries K and K+ngdofs_tgt - // NOTE: add 1 to row/col indices, since e3sm map files indices are 1-based - std::vector col,row,S; - for (int i=0; i creating map file ... done!\n",comm); - - // -------------------------------------- // - // Build src grid and remapper // - // -------------------------------------- // - - print (" -> creating grid and remapper ...\n",comm); - - auto src_grid = build_src_grid(comm, nldofs_src); - - auto remap = std::make_shared(src_grid,filename); - print (" -> creating grid and remapper ... done!\n",comm); - - // -------------------------------------- // - // Create src/tgt grid fields // - // -------------------------------------- // - - print (" -> creating fields ...\n",comm); - constexpr int vec_dim = 3; - - auto tgt_grid = remap->get_tgt_grid(); - // Check that the target grid made by the remapper has the correct number of columns, - // and has the same number of levels as the source grid. - REQUIRE(tgt_grid->get_num_vertical_levels()==src_grid->get_num_vertical_levels()); - REQUIRE(tgt_grid->get_num_global_dofs()==ngdofs_tgt); - - auto src_s2d = create_field("s2d", src_grid,true,false); - auto src_v2d = create_field("v2d", src_grid,true,true); - auto src_s3d_m = create_field("s3d_m",src_grid,false,false,true, 1); - auto src_s3d_i = create_field("s3d_i",src_grid,false,false,false,SCREAM_PACK_SIZE); - auto src_v3d_m = create_field("v3d_m",src_grid,false,true ,true, 1); - auto src_v3d_i = create_field("v3d_i",src_grid,false,true ,false,SCREAM_PACK_SIZE); - - auto tgt_s2d = create_field("s2d", tgt_grid,true,false); - auto tgt_v2d = create_field("v2d", tgt_grid,true,true); - auto tgt_s3d_m = create_field("s3d_m",tgt_grid,false,false,true, 1); - auto tgt_s3d_i = create_field("s3d_i",tgt_grid,false,false,false,SCREAM_PACK_SIZE); - auto tgt_v3d_m = create_field("v3d_m",tgt_grid,false,true ,true, 1); - auto tgt_v3d_i = create_field("v3d_i",tgt_grid,false,true ,false,SCREAM_PACK_SIZE); + auto tgt_grid = remap->get_coarse_grid(); + + auto src_s2d = create_field("s2d", LayoutType::Scalar2D, *src_grid, false, engine); + auto src_v2d = create_field("v2d", LayoutType::Vector2D, *src_grid, false, engine); + auto src_s3d_m = create_field("s3d_m",LayoutType::Scalar3D, *src_grid, true, engine); + auto src_s3d_i = create_field("s3d_i",LayoutType::Scalar3D, *src_grid, false, engine); + auto src_v3d_m = create_field("v3d_m",LayoutType::Vector3D, *src_grid, true, engine); + auto src_v3d_i = create_field("v3d_i",LayoutType::Vector3D, *src_grid, false, engine); + + auto tgt_s2d = create_field("s2d", LayoutType::Scalar2D, *tgt_grid, false); + auto tgt_v2d = create_field("v2d", LayoutType::Vector2D, *tgt_grid, false); + auto tgt_s3d_m = create_field("s3d_m",LayoutType::Scalar3D, *tgt_grid, true ); + auto tgt_s3d_i = create_field("s3d_i",LayoutType::Scalar3D, *tgt_grid, false); + auto tgt_v3d_m = create_field("v3d_m",LayoutType::Vector3D, *tgt_grid, true ); + auto tgt_v3d_i = create_field("v3d_i",LayoutType::Vector3D, *tgt_grid, false); std::vector src_f = {src_s2d,src_v2d,src_s3d_m,src_s3d_i,src_v3d_m,src_v3d_i}; std::vector tgt_f = {tgt_s2d,tgt_v2d,tgt_s3d_m,tgt_s3d_i,tgt_v3d_m,tgt_v3d_i}; - const int nfields = src_f.size(); - - std::vector field_col_size (src_f.size()); - std::vector field_col_offset (src_f.size()+1,0); - for (int i=0; i creating fields ... done!\n",comm); - // -------------------------------------- // // Register fields in the remapper // // -------------------------------------- // - print (" -> registering fields ...\n",comm); remap->registration_begins(); - remap->register_field(src_s2d, tgt_s2d); - remap->register_field(src_v2d, tgt_v2d); - remap->register_field(src_s3d_m,tgt_s3d_m); - remap->register_field(src_s3d_i,tgt_s3d_i); - remap->register_field(src_v3d_m,tgt_v3d_m); - remap->register_field(src_v3d_i,tgt_v3d_i); - remap->registration_ends(); - print (" -> registering fields ... done!\n",comm); - - // -------------------------------------- // - // Check remapper internals // - // -------------------------------------- // - - print (" -> Checking remapper internal state ...\n",comm); - - // Check tgt grid - REQUIRE (tgt_grid->get_num_global_dofs()==ngdofs_tgt); - - // Check which triplets are read from map file - auto src_dofs_h = src_grid->get_dofs_gids().get_view(); - auto my_triplets = remap->test_triplet_gids (filename); - const int num_triplets = my_triplets.size(); - REQUIRE (num_triplets==nnz_local); - for (int i=0; iget_ov_tgt_grid (); - const int num_loc_ov_tgt_gids = ov_tgt_grid->get_num_local_dofs(); - const int expected_num_loc_ov_tgt_gids = ngdofs_tgt>=nldofs_src ? nldofs_src : ngdofs_tgt; - REQUIRE (num_loc_ov_tgt_gids==expected_num_loc_ov_tgt_gids); - const auto ov_gids = ov_tgt_grid->get_dofs_gids().get_view(); - for (int i=0; iget_row_offsets()); - auto col_lids_h = cmvc(remap->get_col_lids()); - auto weights_h = cmvc(remap->get_weights()); - auto ov_tgt_gids = ov_tgt_grid->get_dofs_gids().get_view(); - auto src_gids = remap->get_src_grid()->get_dofs_gids().get_view(); - - REQUIRE (col_lids_h.extent_int(0)==nldofs_src); - REQUIRE (row_offsets_h.extent_int(0)==(num_loc_ov_tgt_gids+1)); - for (int i=0; iregister_field(src_f[i],tgt_f[i]); } - - // Check internal MPI structures - const int num_loc_tgt_gids = tgt_grid->get_num_local_dofs(); - const auto tgt_gids = tgt_grid->get_dofs_gids().get_view(); - const auto recv_lids_beg = remap->get_recv_lids_beg(); - const auto recv_lids_end = remap->get_recv_lids_end(); - const auto recv_lids_pidpos = remap->get_recv_lids_pidpos(); - // Rank 0 sends everything to itself - for (int i=0; i Checking remapper internal state ... OK!\n",comm); + remap->registration_ends(); // -------------------------------------- // - // Generate data for src fields // + // Check remapped fields // // -------------------------------------- // - print (" -> generate src fields data ...\n",comm); - // Generate data in a deterministic way, so that when we check results, - // we know a priori what the input data that generated the tgt field's - // values was, even if that data was off rank. - for (const auto& f : src_f) { - const auto& l = f.get_header().get_identifier().get_layout(); - switch (get_layout_type(l.tags())) { - case LayoutType::Scalar2D: - { - const auto v_src = f.get_view(); - for (int i=0; i(); - for (int i=0; i(); - for (int i=0; i(); - for (int i=0; i generate src fields data ... done!\n",comm); + Real w = 0.5; + auto gids_tgt = all_gather_field(tgt_grid->get_dofs_gids(),comm); + auto gids_src = all_gather_field(src_grid->get_dofs_gids(),comm); + auto gids_src_v = gids_src.get_view(); + auto gids_tgt_v = gids_tgt.get_view(); - auto combine = [] (const Real lhs, const Real rhs) -> Real { - return 0.25*lhs + 0.75*rhs; + auto gid2lid = [&](const int gid, const auto gids_v) { + auto data = gids_v.data(); + auto it = std::find(data,data+gids_v.size(),gid); + return std::distance(data,it); }; - - // No bwd remap - REQUIRE_THROWS(remap->remap(false)); - for (int irun=0; irun<5; ++irun) { - print (" -> run remap ...\n",comm); + root_print (" -> Run " + std::to_string(irun) + "\n",comm); remap->remap(true); - print (" -> run remap ... done!\n",comm); - // -------------------------------------- // - // Check remapped fields // - // -------------------------------------- // - - print (" -> check tgt fields ...\n",comm); - // Recall, tgt gid K should be the avg of src gids K and K+ngdofs_tgt - const int ntgt_gids = tgt_gids.size(); + // Recall, tgt gid K should be the avg of local src_gids for (size_t ifield=0; ifield Checking field with layout " + to_string(l) + " " + dots + "\n",comm); - - f.sync_to_host(); - + auto msg = " -> Checking field with layout " + to_string(l) + " " + dots; + root_print (msg + "\n",comm); + bool ok = true; switch (get_layout_type(l.tags())) { case LayoutType::Scalar2D: { - const auto v_tgt = f.get_view(); - for (int i=0; i(); + const auto v_tgt = gtgt.get_view(); + for (int idof=0; idof(); - for (int i=0; i(); + const auto v_tgt = gtgt.get_view(); + for (int idof=0; idof(); - for (int i=0; i(); + const auto v_tgt = gtgt.get_view(); + auto f_nlevs = gsrc.get_header().get_identifier().get_layout().dims().back(); + for (int idof=0; idof(); - for (int i=0; i(); + const auto v_tgt = gtgt.get_view(); + auto f_nlevs = gsrc.get_header().get_identifier().get_layout().dims().back(); + for (int idof=0; idof Checking field with layout " + to_string(l) + " " + dots + " OK!\n",comm); + root_print (msg + (ok ? "PASS" : "FAIL") + "\n",comm); } - print ("check tgt fields ... done!\n",comm); } // Clean up scorpio stuff diff --git a/components/eamxx/src/share/tests/common_physics_functions_tests.cpp b/components/eamxx/src/share/tests/common_physics_functions_tests.cpp index bac71d2ac71c..c4cff428d676 100644 --- a/components/eamxx/src/share/tests/common_physics_functions_tests.cpp +++ b/components/eamxx/src/share/tests/common_physics_functions_tests.cpp @@ -4,6 +4,7 @@ #include "share/util/scream_setup_random_test.hpp" #include "share/util/scream_common_physics_functions.hpp" +#include "share/util/scream_utils.hpp" #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" @@ -56,14 +57,6 @@ struct ChecksHelpers,NumLevels> { } }; -// Helper function. Create Mirror View and Deep-Copy (CMVDC) -template -auto cmvdc (const ViewT& v_d) -> typename ViewT::HostMirror { - auto v_h = Kokkos::create_mirror_view(v_d); - Kokkos::deep_copy(v_h,v_d); - return v_h; -} - template void run_scalar_valued_fns(std::mt19937_64& engine) { @@ -472,26 +465,26 @@ void run(std::mt19937_64& engine) Kokkos::fence(); // Deep copy to host, and check the properties of the full view output - auto temperature_host = cmvdc(temperature); - auto theta_host = cmvdc(theta); - auto pressure_host = cmvdc(pressure); - auto qv_host = cmvdc(qv); - - auto density_host = cmvdc(density); - auto exner_host = cmvdc(exner); - auto T_from_Theta_host = cmvdc(T_from_Theta); - auto Tv_host = cmvdc(Tv); - auto T_from_Tv_host = cmvdc(T_from_Tv); - auto dse_host = cmvdc(dse); - auto T_from_dse_host = cmvdc(T_from_dse); - auto z_int_host = cmvdc(z_int); - auto dz_host = cmvdc(dz); - auto vmr_host = cmvdc(vmr); - auto mmr_host = cmvdc(mmr); - auto mmr_for_testing_host = cmvdc(mmr_for_testing); - auto wetmmr_host = cmvdc(wetmmr); - auto drymmr_host = cmvdc(drymmr); - auto wetmmr_for_testing_host = cmvdc(wetmmr_for_testing); + auto temperature_host = scream::cmvdc(temperature); + auto theta_host = scream::cmvdc(theta); + auto pressure_host = scream::cmvdc(pressure); + auto qv_host = scream::cmvdc(qv); + + auto density_host = scream::cmvdc(density); + auto exner_host = scream::cmvdc(exner); + auto T_from_Theta_host = scream::cmvdc(T_from_Theta); + auto Tv_host = scream::cmvdc(Tv); + auto T_from_Tv_host = scream::cmvdc(T_from_Tv); + auto dse_host = scream::cmvdc(dse); + auto T_from_dse_host = scream::cmvdc(T_from_dse); + auto z_int_host = scream::cmvdc(z_int); + auto dz_host = scream::cmvdc(dz); + auto vmr_host = scream::cmvdc(vmr); + auto mmr_host = scream::cmvdc(mmr); + auto mmr_for_testing_host = scream::cmvdc(mmr_for_testing); + auto wetmmr_host = scream::cmvdc(wetmmr); + auto drymmr_host = scream::cmvdc(drymmr); + auto wetmmr_for_testing_host = scream::cmvdc(wetmmr_for_testing); for (int k=0; k + +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "share/field/field_utils.hpp" +#include "share/field/field.hpp" +#include "share/field/field_manager.hpp" + +#include "share/util/eamxx_time_interpolation.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_time_stamp.hpp" + +#include "share/io/scream_output_manager.hpp" + +#include "ekat/ekat_parameter_list.hpp" +/*----------------------------------------------------------------------------------------------- + * Test TimeInterpolation class + *-----------------------------------------------------------------------------------------------*/ + +namespace scream { + +// Test Constants +constexpr Real tol = std::numeric_limits::epsilon()*1e5; +constexpr int slp_switch = 4; // The frequency that we change the slope of the data written to file. +constexpr int dt = 100; +constexpr int total_snaps = 10; +constexpr int snap_freq = 4; +constexpr int slope_freq = snap_freq; // We will change the slope every slope_freq steps to ensure that the data is not all on the same line. +constexpr int snaps_per_file = 3; + +// Functions needed to set the test up +std::shared_ptr get_gm (const ekat::Comm& comm, const int ncols, const int nlevs); +std::shared_ptr get_fm (const std::shared_ptr& grid, const util::TimeStamp& t0, const int seed); +util::TimeStamp init_timestamp(); +std::vector create_test_data_files(const ekat::Comm& comm, const std::shared_ptr& gm,const util::TimeStamp& t0, const int seed); + +// Functions needed to run the test +void update_field_data(const Real slope, const Real dt, Field& field); +bool views_are_approx_equal(const Field& f0, const Field& f1, const Real tol); + +// Helper randomizer +Real my_pdf(std::mt19937_64& engine) { + std::uniform_int_distribution pdf (1,100); + Real v = pdf(engine); + return v; +}; + +// Wrapper for output manager that also extracts the list of files +class OutputManager4Test : public scream::OutputManager +{ +public: + OutputManager4Test() + : OutputManager() + { + // Do Nothing + } + + void runme(const util::TimeStamp& ts) { + run(ts); + update_file_list(); + } + + std::vector get_list_of_files() { return m_list_of_files; } +private: + void update_file_list() { + if (std::find(m_list_of_files.begin(),m_list_of_files.end(), m_output_file_specs.filename) == m_list_of_files.end()) { + m_list_of_files.push_back(m_output_file_specs.filename); + } + } + std::vector m_list_of_files; +}; +/*-----------------------------------------------------------------------------------------------*/ +TEST_CASE ("eamxx_time_interpolation_simple") { + printf("TimeInterpolation - Simple Case...\n\n\n"); + // Setup basic test params + ekat::Comm comm(MPI_COMM_WORLD); + auto seed = get_random_test_seed(&comm); + std::mt19937_64 engine(seed); + const auto t0 = init_timestamp(); + + const int nlevs = SCREAM_PACK_SIZE*2+1; + const int ncols = comm.size()*2 + 1; + + // Get a grids manager for the test + auto grids_man = get_gm(comm, ncols, nlevs); + const auto& grid = grids_man->get_grid("Point Grid"); + // Now create a fields manager to store initial data for testing. + auto fields_man_t0 = get_fm(grid, t0, seed); + // Construct a time interpolation object and add all of the fields to it. + printf( "Constructing a time interpolation object ...\n"); + util::TimeInterpolation time_interpolator(grid); + for (auto ff_pair = fields_man_t0->begin(); ff_pair != fields_man_t0->end(); ff_pair++) { + const auto ff = ff_pair->second; + time_interpolator.add_field(*ff); + time_interpolator.initialize_data_from_field(*ff); + } + time_interpolator.initialize_timestamps(t0); + printf( "Constructing a time interpolation object ... DONE\n"); + + // Set the field data for a step 10 dt in the future to be used for interpolation. We + // create a new field manager so we can continue to have the intial condition for reference. + printf( "Setting data for other end of time interpolation, at t = 10 dt ...\n"); + auto slope = my_pdf(engine); + auto fields_man_tf = get_fm(grid, t0, seed); + auto t1 = t0 + 10*dt; + for (auto ff_pair = fields_man_tf->begin(); ff_pair != fields_man_tf->end(); ff_pair++) + { + auto ff = ff_pair->second; + update_field_data(slope, t1.seconds_from(t0), *ff); + time_interpolator.update_data_from_field(*ff); + } + time_interpolator.update_timestamp(t1); + printf( "Setting data for other end of time interpolation, at t = 10 dt ...DONE\n"); + + // Now check that the interpolator is working as expected. Should be able to + // match the interpolated fields against the results of update_field_data at any + // time between 0 and 10 dt. + printf( "Testing all timesteps ... slope = %f\n",slope); + auto fields_man_test = get_fm(grid, t0, seed); + t1 = t0; + for (int tt = 1; tt<=10; tt++) { + t1 += dt; + printf(" ... t = %s\n",t1.to_string().c_str()); + time_interpolator.perform_time_interpolation(t1); + for (auto ff_pair = fields_man_test->begin(); ff_pair != fields_man_test->end(); ff_pair++) + { + const auto name = ff_pair->first; + auto ff = ff_pair->second; + update_field_data(slope, dt, *ff); + REQUIRE(views_are_approx_equal(*ff,time_interpolator.get_field(name),tol)); + } + } + printf(" ... DONE\n"); + printf("TimeInterpolation - Simple Case...DONE\n\n\n"); +} // TEST_CASE eamxx_time_interpolation_simple +/*-----------------------------------------------------------------------------------------------*/ +TEST_CASE ("eamxx_time_interpolation_data_from_file") { + printf("TimeInterpolation - From File Case...\n\n\n"); + // Setup basic test + printf(" - Test Basics...\n"); + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); + auto seed = get_random_test_seed(&comm); + std::mt19937_64 engine(seed); + const auto t0 = init_timestamp(); + + const int nlevs = SCREAM_PACK_SIZE*2+1; + const int ncols = comm.size()*2 + 1; + printf(" - Test Basics...DONE\n"); + + // Get a grids manager for the test + printf(" - Grids Manager...\n"); + auto grids_man = get_gm(comm, ncols, nlevs); + const auto& grid = grids_man->get_grid("Point Grid"); + printf(" - Grids Manager...DONE\n"); + // Now create a fields manager to store initial data for testing. + printf(" - Fields Manager...\n"); + auto fields_man_t0 = get_fm(grid, t0, seed); + auto fields_man_deep = get_fm(grid, t0, seed); // A field manager for checking deep copies. + std::vector fnames; + for (auto it : *fields_man_t0) { + fnames.push_back(it.second->name()); + } + printf(" - Fields Manager...DONE\n"); + // Construct the files of interpolation data + printf(" - create test data files...\n"); + auto list_of_files = create_test_data_files(comm, grids_man, t0, seed); + printf(" - create test data files...DONE\n"); + + // Construct a time interpolation object using the list of files with the data + printf( "Constructing a time interpolation object ...\n"); + util::TimeInterpolation time_interpolator(grid,list_of_files); + util::TimeInterpolation time_interpolator_deep(grid,list_of_files); + for (auto name : fnames) { + auto ff = fields_man_t0->get_field(name); + auto ff_deep = fields_man_deep->get_field(name); + time_interpolator.add_field(ff); + time_interpolator_deep.add_field(ff_deep,true); + } + time_interpolator.initialize_data_from_files(); + time_interpolator_deep.initialize_data_from_files(); + printf( "Constructing a time interpolation object ... DONE\n"); + + // Now check that the interpolator is working as expected. Should be able to + // match the interpolated fields against the results of update_field_data at any + // time between 0 and 10 dt. + printf( "Testing all timesteps ...\n"); + const int max_steps = snap_freq*total_snaps; + // The strategy is to update the model state following the same linear updates + // that we used to generate the time interpolation data. The fields produced + // by the time interpolator should match. + auto ts = t0; + for (int nn=0; nn<=max_steps; nn++) { + // Update Time and slope + if (nn > 0) { + ts += dt; + } + printf(" ... t = %s\n",ts.to_string().c_str()); + // Perform time interpolation + if (nn > 0) { + const Real slope = ((nn-1) / slope_freq) + 1; + for (auto name : fnames) { + auto field = fields_man_t0->get_field(name); + update_field_data(slope,dt,field); + // We set the deep copy fields to wrong values to stress test that everything still works. + auto field_deep = fields_man_deep->get_field(name); + field_deep.deep_copy(-9999.0); + } + } + time_interpolator.perform_time_interpolation(ts); + time_interpolator_deep.perform_time_interpolation(ts); + // Now compare the interp_fields to the fields in the field manager which should be updated. + for (auto name : fnames) { + auto field = fields_man_t0->get_field(name); + auto field_deep = fields_man_deep->get_field(name); + // Check that the shallow copies match the expected values + REQUIRE(views_are_approx_equal(field,time_interpolator.get_field(name),tol)); + // Check that the deep fields which were not updated directly are the same as the ones stored in the time interpolator. + REQUIRE(views_are_equal(field_deep,time_interpolator_deep.get_field(name))); + // Check that the deep and shallow fields match showing that both approaches got the correct answer. + REQUIRE(views_are_equal(field,field_deep)); + } + + } + + + time_interpolator.finalize(); + time_interpolator_deep.finalize(); + printf(" ... DONE\n"); + + // All done with IO + scorpio::eam_pio_finalize(); + + printf("TimeInterpolation - From File Case...DONE\n\n\n"); +} // TEST_CASE eamxx_time_interpolation_data_from_file +/*-----------------------------------------------------------------------------------------------*/ +/*-----------------------------------------------------------------------------------------------*/ + +/*-----------------------------------------------------------------------------------------------*/ +bool views_are_approx_equal(const Field& f0, const Field& f1, const Real tol) +{ + const auto& l0 = f0.get_header().get_identifier().get_layout(); + const auto& l1 = f1.get_header().get_identifier().get_layout(); + EKAT_REQUIRE_MSG(l0==l1,"Error! views_are_approx_equal - the two fields don't have matching layouts."); + // Take advantage of field utils update, min and max to assess the max difference between the two fields + // simply. + auto ft = f0.clone(); + ft.update(f1,1.0,-1.0); + auto d_min = field_min(ft); + auto d_max = field_max(ft); + if (std::abs(d_min) > tol or std::abs(d_max) > tol) { + printf("The two copies of (%16s) are NOT approx equal within a tolerance of %e.\n The min and max errors are %e and %e respectively.\n",f0.name().c_str(),tol,d_min,d_max); + return false; + } else { + return true; + } + +} +/*-----------------------------------------------------------------------------------------------*/ +void update_field_data(const Real slope, const Real dt, Field& field) +{ + const auto& l1 = field.get_header().get_identifier().get_layout(); + const auto& dims = l1.dims(); + switch (l1.rank()) { + case 1: + { + auto view = field.template get_view(); + for (int ii=0; ii(); + for (int ii=0; ii(); + for (int ii=0; ii get_gm (const ekat::Comm& comm, const int ncols, const int nlevs) +{ + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", ncols); + pl.set("number_of_vertical_levels", nlevs); + auto gm = create_mesh_free_grids_manager(comm,gm_params); + gm->build_grids(); + return gm; +} +/*-----------------------------------------------------------------------------------------------*/ +/* Create a fields manager for the test */ +std::shared_ptr get_fm (const std::shared_ptr& grid, const util::TimeStamp& t0, const int seed) +{ + using FL = FieldLayout; + using FID = FieldIdentifier; + using namespace ShortFieldTagsNames; + + // Random number generation stuff + // NOTES + // - Use integers, so we can check answers without risk of + // non bfb diffs due to different order of sums. + // - Uniform_int_distribution returns an int, and the randomize + // util checks that return type matches the Field data type. + // So wrap the int pdf in a lambda, that does the cast. + std::mt19937_64 engine(seed); + + const int nlcols = grid->get_num_local_dofs(); + const int nlevs = grid->get_num_vertical_levels(); + + std::vector layouts = + { + FL({COL }, {nlcols }), + FL({COL, LEV}, {nlcols, nlevs}), + FL({COL,CMP,ILEV}, {nlcols,2,nlevs+1}) + }; + + auto fm = std::make_shared(grid); + fm->registration_begins(); + fm->registration_ends(); + + const auto units = ekat::units::Units::nondimensional(); + for (const auto& fl : layouts) { + FID fid("f_"+std::to_string(fl.size()),fl,units,grid->name()); + Field f(fid); + f.allocate_view(); + randomize (f,engine,my_pdf); + f.get_header().get_tracking().update_time_stamp(t0); + fm->add_field(f); + } + + return fm; +} +/*-----------------------------------------------------------------------------------------------*/ +/* Construct data for multiple time snaps and write the data to file, to be used for testing + * the capability of TimeInterpolation to handle data read from multiple files. + */ +std::vector create_test_data_files( + const ekat::Comm& comm, + const std::shared_ptr& gm, + const util::TimeStamp& t0, + const int seed) +{ + // We initialize a local field manager to use for output + auto fm = get_fm(gm->get_grid("Point Grid"), t0, seed); + // We will write data for 10 snaps for this test. We set the max snaps per file to 3 to + // ensure that a) there is more than 1 file and b) at least one file has fewer snap then + // the others. + const int max_steps = snap_freq*total_snaps; + // Gather the set of fields from the field manager + std::vector fnames; + for (auto it : *fm) { + fnames.push_back(it.second->name()); + } + // Create the output parameters + ekat::ParameterList om_pl; + om_pl.set("MPI Ranks in Filename",true); + om_pl.set("filename_prefix",std::string("source_data_for_time_interpolation")); + om_pl.set("Field Names",fnames); + om_pl.set("Averaging Type", std::string("INSTANT")); + om_pl.set("Max Snapshots Per File",snaps_per_file); + auto& ctrl_pl = om_pl.sublist("output_control"); + ctrl_pl.set("frequency_units",std::string("nsteps")); + ctrl_pl.set("Frequency",snap_freq); + ctrl_pl.set("MPI Ranks in Filename",true); + ctrl_pl.set("save_grid_data",false); + // Create an output manager, note we use a subclass defined in this test so we can extract + // the list of files created by the output manager. + OutputManager4Test om; + om.setup(comm,om_pl,fm,gm,t0,false); + + // Time loop to create and write data + auto tw = t0; + for (int nn=0; nn<=max_steps; nn++) { + // Update Time and slope + tw += dt; + const Real slope = (nn / slope_freq) + 1; + // Update Fields + for (auto ff : fnames) { + auto field = fm->get_field(ff); + update_field_data(slope,dt,field); + } + // Run output manager + om.runme(tw); + } + + // Now that all the data is written we finalize everything and return the list of files. + om.finalize(); + + // Jumble the order of files to also test the sorting algorithm + auto list_of_files_tmp = om.get_list_of_files(); + std::vector list_of_files; + for (size_t ii = 1; ii()); // But this should work - REQUIRE_NOTHROW(f1.get_view()); + f1.get_view(); // Using packs (of allowable size) of different pack sizes // should lead to views with different extents. @@ -162,6 +147,26 @@ TEST_CASE("field", "") { REQUIRE(views_are_equal(f1,f2)); } + SECTION ("construct_from_view") { + // Crate f1 with some padding, to stress test the feature + Field f1 (fid); + auto& fap1 = f1.get_header().get_alloc_properties(); + fap1.request_allocation(16); + f1.allocate_view(); + f1.deep_copy(1.0); + + // Get f1 view, and wrap it in another field + auto view = f1.get_view(); + Field f2 (fid,view); + + // Check the two are the same + REQUIRE (views_are_equal(f1,f2)); + + // Modify one field, and check again + randomize(f2,engine,pdf); + REQUIRE (views_are_equal(f1,f2)); + } + SECTION ("clone") { Field f1 (fid); auto& fap1 = f1.get_header().get_alloc_properties(); @@ -184,6 +189,26 @@ TEST_CASE("field", "") { REQUIRE (field_min(f1)==3.0); } + SECTION ("alias") { + Field f1 (fid); + f1.allocate_view(); + + Field f2 = f1.alias("the_alias"); + + REQUIRE(f2.is_allocated()); + REQUIRE(&f1.get_header().get_tracking()==&f2.get_header().get_tracking()); + REQUIRE(&f1.get_header().get_alloc_properties()==&f2.get_header().get_alloc_properties()); + REQUIRE(f1.get_header().get_identifier().get_layout()==f2.get_header().get_identifier().get_layout()); + REQUIRE(f1.get_internal_view_data()==f2.get_internal_view_data()); + + // Identifiers are separate objects though + REQUIRE(&f1.get_header().get_identifier()!=&f2.get_header().get_identifier()); + + // Check extra data is also shared + f1.get_header().set_extra_data("foo",1); + REQUIRE (f2.get_header().has_extra_data("foo")); + } + SECTION ("deep_copy") { std::vector t1 = {COL,CMP,LEV}; std::vector d1 = {3,2,24}; @@ -330,6 +355,39 @@ TEST_CASE("field", "") { } } } + + SECTION ("rank0_field") { + // Create 0d field + FieldIdentifier fid0("f_0d", FieldLayout({},{}), Units::nondimensional(), "dummy_grid"); + Field f0(fid0); + f0.allocate_view(); + + // Create 1d field + FieldIdentifier fid1("f_1d", FieldLayout({COL}, {5}), Units::nondimensional(), "dummy_grid"); + Field f1(fid1); + f1.allocate_view(); + + // Randomize 1d field + randomize(f1,engine,pdf); + + auto v0 = f0.get_view(); + auto v1 = f1.get_view(); + + // Deep copy subfield of 1d field -> 0d field and check result + for (size_t i=0; i(f1.subfield(0, i)); + REQUIRE(v0() == v1(i)); + } + + // Randomize 0d field + randomize(f0,engine,pdf); + + // Deep copy 0d field -> subfield of 1d field and check result + for (size_t i=0; i(f0); + REQUIRE(v1(i) == v0()); + } + } } TEST_CASE("field_group") { @@ -578,9 +636,9 @@ TEST_CASE("tracers_bundle", "") { int idx_v, idx_c, idx_r; // The idx must be stored - REQUIRE_NOTHROW (idx_v = group.m_info->m_subview_idx.at("qv")); - REQUIRE_NOTHROW (idx_c = group.m_info->m_subview_idx.at("qc")); - REQUIRE_NOTHROW (idx_r = group.m_info->m_subview_idx.at("qr")); + idx_v = group.m_info->m_subview_idx.at("qv"); + idx_c = group.m_info->m_subview_idx.at("qc"); + idx_r = group.m_info->m_subview_idx.at("qr"); // All idx must be in [0,2] and must be different REQUIRE ((idx_v>=0 && idx_v<3 && @@ -793,7 +851,17 @@ TEST_CASE ("update") { f3.update(f_real,2,0); REQUIRE (views_are_equal(f3,f2)); } -} + SECTION ("scale") { + Field f1 = f_real.clone(); + Field f2 = f_real.clone(); + + // x=2, x*y = 2*y + f1.deep_copy(2.0); + f1.scale(f2); + f2.scale(2.0); + REQUIRE (views_are_equal(f1, f2)); + } +} } // anonymous namespace diff --git a/components/eamxx/src/share/tests/field_utils.cpp b/components/eamxx/src/share/tests/field_utils.cpp index 375bf184ec64..b8c901c9f390 100644 --- a/components/eamxx/src/share/tests/field_utils.cpp +++ b/components/eamxx/src/share/tests/field_utils.cpp @@ -22,6 +22,7 @@ TEST_CASE("utils") { using namespace ShortFieldTagsNames; using namespace ekat::units; using kt = KokkosTypes; + using kt_host = KokkosTypes; using P8 = ekat::Pack; @@ -172,6 +173,103 @@ TEST_CASE("utils") { REQUIRE(field_min(f1,&comm)==gmin); } + SECTION ("perturb") { + using namespace ShortFieldTagsNames; + using RPDF = std::uniform_real_distribution; + using IPDF = std::uniform_int_distribution; + auto engine = setup_random_test (); + + const int ncols = 6; + const int ncmps = 2; + const int nlevs = IPDF(3,9)(engine); // between 3-9 levels + + // Create 1d, 2d, 3d fields with a level dimension, and set all to 1 + FieldIdentifier fid1 ("f_1d", FieldLayout({LEV}, {nlevs}), Units::nondimensional(), ""); + FieldIdentifier fid2a("f_2d_a", FieldLayout({CMP, LEV}, {ncmps, nlevs}), Units::nondimensional(), ""); + FieldIdentifier fid2b("f_2d_b", FieldLayout({COL, LEV}, {ncols, nlevs}), Units::nondimensional(), ""); + FieldIdentifier fid3 ("f_3d", FieldLayout({COL, CMP, LEV}, {ncols, ncmps, nlevs}), Units::nondimensional(), ""); + Field f1(fid1), f2a(fid2a), f2b(fid2b), f3(fid3); + f1.allocate_view(), f2a.allocate_view(), f2b.allocate_view(), f3.allocate_view(); + f1.deep_copy(1), f2a.deep_copy(1), f2b.deep_copy(1), f3.deep_copy(1); + + // We need GIDs for fields with COL component. This test is not over + // multiple ranks, so just set as [0, ncols-1]. + Field gids(FieldIdentifier("gids", FieldLayout({COL}, {ncols}), Units::nondimensional(), "", DataType::IntType)); + gids.allocate_view(); + auto gids_data = gids.get_internal_view_data(); + std::iota(gids_data, gids_data+ncols, 0); + gids.sync_to_dev(); + + // Create masks s.t. only last 3 levels are perturbed. For variety, + // 1d and 2d fields will use lambda mask and 3 field will use a view. + auto mask_lambda = [&nlevs] (const int& i0) { + return i0 >= nlevs-3; + }; + kt_host::view_1d mask_view("mask_view", nlevs); + Kokkos::deep_copy(mask_view, false); + for (int ilev=0; ilev= nlevs-3) mask_view(ilev) = true; + } + + // Compute random perturbation between [2, 3] + RPDF pdf(2, 3); + int base_seed = 0; + perturb(f1, engine, pdf, base_seed, mask_lambda); + perturb(f2a, engine, pdf, base_seed, mask_lambda); + perturb(f2b, engine, pdf, base_seed, mask_lambda, gids); + perturb(f3, engine, pdf, base_seed, mask_view, gids); + + // Sync to host for checks + f1.sync_to_host(), f2a.sync_to_host(), f2b.sync_to_host(), f3.sync_to_host(); + const auto v1 = f1.get_view (); + const auto v2a = f2a.get_view(); + const auto v2b = f2b.get_view(); + const auto v3 = f3.get_view (); + + // Check that all field values are 1 for all but last 3 levels and between [2,3] otherwise. + auto check_level = [&] (const int ilev, const Real val) { + if (ilev < nlevs-3) REQUIRE(val == 1); + else REQUIRE((2 <= val && val <= 3)); + }; + for (int icol=0; icol(); + const auto v3_alt = f3_alt.get_view(); + + auto check_diff = [&] (const int ilev, const Real val1, const Real val2) { + if (ilev < nlevs-3) REQUIRE(val1==val2); + else REQUIRE(val1!=val2); + }; + for (int icol=0; icol::value, diff --git a/components/eamxx/src/share/tests/grid_import_export_tests.cpp b/components/eamxx/src/share/tests/grid_import_export_tests.cpp new file mode 100644 index 000000000000..4a0d04358d75 --- /dev/null +++ b/components/eamxx/src/share/tests/grid_import_export_tests.cpp @@ -0,0 +1,201 @@ +// Access some internal catch2 types. +// #define CATCH_CONFIG_EXTERNAL_INTERFACES +#include + +#include "share/grid/point_grid.hpp" +#include "share/grid/grid_import_export.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/scream_types.hpp" + +#include + +namespace { + +using namespace scream; +using namespace scream::ShortFieldTagsNames; + +TEST_CASE ("grid_import_export") { + using gid_type = AbstractGrid::gid_type; + + ekat::Comm comm(MPI_COMM_WORLD); + MPI_Comm_set_errhandler(comm.mpi_comm(),MPI_ERRORS_RETURN); + + auto& catch_capture = Catch::getResultCapture(); + + // return; + auto engine = setup_random_test(&comm); + + bool ok; + const int overlap = 5; + const int nldofs_src = 10; + const int ngdofs = nldofs_src*comm.size();; + + // Create the unique grid + auto src_grid = create_point_grid("src",ngdofs,0,comm); + auto src_gids = src_grid->get_dofs_gids().get_view(); + + // For the dst grid, shuffle dofs around randomly. Then, + // have each rank grab a few extra dofs + std::vector all_dofs (ngdofs); + if (comm.am_i_root()) { + std::iota(all_dofs.data(),all_dofs.data()+all_dofs.size(),0); + std::shuffle(all_dofs.data(),all_dofs.data()+ngdofs,engine); + } + comm.broadcast(all_dofs.data(),ngdofs,comm.root_rank()); + + const bool first = comm.rank()==0; + const bool last = comm.rank()==(comm.size()-1); + auto start = all_dofs.data() + nldofs_src*comm.rank(); + auto end = start + nldofs_src; + end += last ? 0 : overlap; + start -= first ? 0 : overlap; + const int ndofs_dst = nldofs_src + (first ? 0 : overlap) + (last ? 0 : overlap); + + auto dst_grid = std::make_shared("grid",ndofs_dst,0,comm); + auto dst_gids_field = dst_grid->get_dofs_gids(); + auto dst_gids = dst_gids_field.get_view(); + + std::copy (start,end,dst_gids.data()); + dst_gids_field.sync_to_dev(); + + GridImportExport imp_exp(src_grid,dst_grid); + + // Test import views + if (comm.am_i_root()) { + printf(" -> Testing import views ......\n"); + } + ok = true; + auto imp_pids = imp_exp.import_pids_h(); + auto imp_lids = imp_exp.import_lids_h(); + std::set all_imp_lids; + for (size_t i=0; i Testing import views ...... %s\n",ok ? "PASS" : "FAIL"); + } + + // Test export views + if (comm.am_i_root()) { + printf(" -> Testing export views ......\n"); + } + ok = true; + auto exp_pids = imp_exp.export_pids_h(); + auto exp_lids = imp_exp.export_lids_h(); + std::map> pid2lid; + auto src_gid2lid = src_grid->get_gid2lid_map(); + for (int pid=0; pid Testing export views ...... %s\n",ok ? "PASS" : "FAIL"); + } + + // Test gather + if (comm.am_i_root()) { + printf(" -> Testing gather routine ....\n"); + } + ok = true; + std::map> src_data; + for (int i=0; iget_num_local_dofs(); ++i) { + auto& v = src_data[i]; + auto gid = dst_gids[i]; + v.resize(gid); + std::iota(v.begin(),v.end(),0); + } + std::map> dst_data; + imp_exp.gather(ekat::get_mpi_type(),src_data,dst_data); + + std::vector num_imp_per_lid (nldofs_src,0); + for (size_t i=0; i expected(gid); + std::iota(expected.begin(),expected.end(),0); + for (size_t imp=0; imp Testing gather routine .... %s\n",ok ? "PASS" : "FAIL"); + } + + // Test scatter + if (comm.am_i_root()) { + printf(" -> Testing scatter routine ...\n"); + } + ok = true; + src_data.clear(); + // std::cout << "src_data:\n"; + for (int i=0; iget_num_local_dofs(); ++i) { + auto& v = src_data[i]; + auto gid = src_gids[i]; + v.resize(gid); + std::iota(v.begin(),v.end(),0); + // std::cout << " " << gid << ":" << ekat::join(v," ") << "\n"; + } + dst_data.clear(); + imp_exp.scatter(ekat::get_mpi_type(),src_data,dst_data); + + // std::cout << "dst_data:\n"; + for (const auto& it : dst_data) { + auto lid = it.first; + auto gid = dst_gids[lid]; + const auto& v = it.second; + CHECK (static_cast(v.size())==gid); + ok &= catch_capture.lastAssertionPassed(); + // std::cout << " " << gid << ":" << ekat::join(v," ") << "\n"; + + std::vector expected(gid); + std::iota(expected.begin(),expected.end(),0); + CHECK (std::equal(expected.begin(),expected.end(),v.begin())); + ok &= catch_capture.lastAssertionPassed(); + } + if (comm.am_i_root()) { + printf(" -> Testing scatter routine ... %s\n",ok ? "PASS" : "FAIL"); + } +} + +} // anonymous namespace diff --git a/components/eamxx/src/share/tests/grid_tests.cpp b/components/eamxx/src/share/tests/grid_tests.cpp index 1162c92b8983..b9a7b81cf731 100644 --- a/components/eamxx/src/share/tests/grid_tests.cpp +++ b/components/eamxx/src/share/tests/grid_tests.cpp @@ -122,9 +122,9 @@ TEST_CASE ("get_owners") { auto grid = std::make_shared("grid",num_local_dofs,2,comm); // Create dofs, shuffled them around across ranks. - using gid_t = AbstractGrid::gid_type; + using gid_type = AbstractGrid::gid_type; - std::vector all_dofs (num_global_dofs); + std::vector all_dofs (num_global_dofs); if (comm.am_i_root()) { std::iota(all_dofs.data(),all_dofs.data()+all_dofs.size(),0); std::shuffle(all_dofs.data(),all_dofs.data()+num_global_dofs,engine); @@ -132,8 +132,8 @@ TEST_CASE ("get_owners") { comm.broadcast(all_dofs.data(),num_global_dofs,comm.root_rank()); auto dofs = grid->get_dofs_gids(); - auto dofs_h = dofs.get_view(); - std::memcpy (dofs_h.data(),all_dofs.data()+offset,num_local_dofs*sizeof(gid_t)); + auto dofs_h = dofs.get_view(); + std::memcpy (dofs_h.data(),all_dofs.data()+offset,num_local_dofs*sizeof(gid_type)); dofs.sync_to_dev(); // Now, ask each rank to retrieve owners, and verify @@ -147,4 +147,75 @@ TEST_CASE ("get_owners") { } } +TEST_CASE ("gid2lid_map") { + using gid_type = AbstractGrid::gid_type; + + ekat::Comm comm(MPI_COMM_WORLD); + + auto engine = setup_random_test(&comm); + + const int num_local_dofs = 10; + const int num_global_dofs = num_local_dofs*comm.size();; + // Create dofs, shuffled them around across ranks. + std::vector all_dofs (num_global_dofs); + if (comm.am_i_root()) { + std::iota(all_dofs.data(),all_dofs.data()+all_dofs.size(),0); + std::shuffle(all_dofs.data(),all_dofs.data()+num_global_dofs,engine); + } + comm.broadcast(all_dofs.data(),num_global_dofs,comm.root_rank()); + + // Create a grid, grabbing my portion of the all_dofs array + const int offset = num_local_dofs*comm.rank(); + auto grid = std::make_shared("grid",num_local_dofs,0,comm); + auto dofs = grid->get_dofs_gids(); + auto dofs_h = dofs.get_view(); + std::memcpy (dofs_h.data(),all_dofs.data()+offset,num_local_dofs*sizeof(gid_type)); + dofs.sync_to_dev(); + + auto gid2lid = grid->get_gid2lid_map(); + for (const auto& it : gid2lid) { + REQUIRE (it.first==dofs_h[it.second]); + } +} + +TEST_CASE ("get_remote_pids_and_lids") { + using gid_type = AbstractGrid::gid_type; + + ekat::Comm comm(MPI_COMM_WORLD); + + auto engine = setup_random_test(&comm); + + const int num_local_dofs = 10; + const int num_global_dofs = num_local_dofs*comm.size();; + // Create dofs, shuffled them around across ranks. + std::vector all_dofs (num_global_dofs); + if (comm.am_i_root()) { + std::iota(all_dofs.data(),all_dofs.data()+all_dofs.size(),0); + std::shuffle(all_dofs.data(),all_dofs.data()+num_global_dofs,engine); + } + comm.broadcast(all_dofs.data(),num_global_dofs,comm.root_rank()); + + // Create a grid, grabbing my portion of the all_dofs array + const int offset = num_local_dofs*comm.rank(); + auto grid = std::make_shared("grid",num_local_dofs,0,comm); + auto dofs = grid->get_dofs_gids(); + auto dofs_h = dofs.get_view(); + std::memcpy (dofs_h.data(),all_dofs.data()+offset,num_local_dofs*sizeof(gid_type)); + dofs.sync_to_dev(); + + // Now, ask each rank to retrieve owners and local ids, and verify + std::vector pids, lids; + grid->get_remote_pids_and_lids(all_dofs,pids,lids); + REQUIRE (pids.size()==all_dofs.size()); + REQUIRE (lids.size()==all_dofs.size()); + + for (int i=0; i get_test_gm(const ekat::Comm& comm, const Int num_gcols, const Int num_levs) { /* Simple routine to construct and return a grids manager given number of columns and levels */ - ekat::ParameterList gm_params; - gm_params.set("number_of_global_columns",num_gcols); - gm_params.set("number_of_vertical_levels",num_levs); - auto gm = create_mesh_free_grids_manager(comm,gm_params); + auto gm = create_mesh_free_grids_manager(comm,0,0,num_levs,num_gcols); gm->build_grids(); return gm; } // end get_test_gm diff --git a/components/eamxx/src/share/tests/property_checks.cpp b/components/eamxx/src/share/tests/property_checks.cpp index c41ca38e7dc9..1cb5e64369ed 100644 --- a/components/eamxx/src/share/tests/property_checks.cpp +++ b/components/eamxx/src/share/tests/property_checks.cpp @@ -43,7 +43,7 @@ TEST_CASE("property_check_base", "") { REQUIRE_THROWS (std::make_shared(cf,grid,0,true)); // But ok if no repair is needed - REQUIRE_NOTHROW (std::make_shared(cf,grid,0,false)); + std::make_shared(cf,grid,0,false); } } @@ -83,15 +83,24 @@ TEST_CASE("property_checks", "") { // Create a field std::vector tags = {COL, CMP, LEV}; std::vector dims = {num_lcols, 3, nlevs}; - FieldIdentifier fid ("field_1",{tags,dims}, m/s,"some_grid"); + FieldIdentifier fid ("field_1", {tags,dims}, m/s,"some_grid"); Field f(fid); f.allocate_view(); + // Create additional data + std::vector tags_data = {COL}; + std::vector dims_data = {num_lcols}; + FieldIdentifier data_fid("data", {tags_data, dims_data}, m/s, "some_grid"); + Field data(data_fid); + data.allocate_view(); + data.deep_copy(1.0); + // Check that values are not NaN SECTION("field_not_nan_check") { const auto num_reals = f.get_header().get_alloc_properties().get_num_scalars(); auto nan_check = std::make_shared(f,grid); + nan_check->set_additional_data_field(data); // Assign values to the field and make sure it passes our test for NaNs. auto f_data = reinterpret_cast(f.get_internal_view_data()); @@ -106,11 +115,18 @@ TEST_CASE("property_checks", "") { f.sync_to_dev(); res_and_msg = nan_check->check(); REQUIRE(res_and_msg.result==CheckResult::Fail); + std::string expected_msg = "FieldNaNCheck failed.\n" " - field id: " + fid.get_id_string() + "\n" - " - entry (1,2,3)\n" - " - lat/lon: (1.000000, -1.000000)\n"; + " - indices (w/ global column index): (1,2,3)\n" + " - lat/lon: (1.000000, -1.000000)\n" + " - additional data (w/ local column index):\n\n" + " data(2)\n\n"+ + " data(1)\n"+ + " 1, \n\n"+ + " END OF ADDITIONAL DATA\n"; + REQUIRE( res_and_msg.msg == expected_msg ); } @@ -121,6 +137,8 @@ TEST_CASE("property_checks", "") { auto interval_check = std::make_shared(f, grid, 0, 1, true); REQUIRE(interval_check->can_repair()); + interval_check->set_additional_data_field(data); + // Assign in-bound values to the field and make sure it passes the within-interval check auto f_data = reinterpret_cast(f.get_internal_view_data()); ekat::genRandArray(f_data,num_reals,engine,pos_pdf); @@ -143,11 +161,11 @@ TEST_CASE("property_checks", "") { " - field id: " + fid.get_id_string() + "\n" " - minimum:\n" " - value: 0\n" - " - entry: (0,1,2)\n" + " - indices (w/ global column index): (0,1,2)\n" " - lat/lon: (0, 0)\n" " - maximum:\n" " - value: 2\n" - " - entry: (1,2,3)\n" + " - indices (w/ global column index): (1,2,3)\n" " - lat/lon: (1, -1)\n"; REQUIRE(res_and_msg.msg == expected_msg); @@ -191,11 +209,11 @@ TEST_CASE("property_checks", "") { " - field id: " + fid.get_id_string() + "\n" " - minimum:\n" " - value: -2\n" - " - entry: (0,0,0)\n" + " - indices (w/ global column index): (0,0,0)\n" " - lat/lon: (0, 0)\n" " - maximum:\n" " - value: 3\n" - " - entry: (1,2,11)\n" + " - indices (w/ global column index): (1,2,11)\n" " - lat/lon: (1, -1)\n"; REQUIRE(res_and_msg.msg == expected_msg); diff --git a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp new file mode 100644 index 000000000000..00ee58ef503c --- /dev/null +++ b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp @@ -0,0 +1,424 @@ +#include + +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/grid/point_grid.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_utils.hpp" +#include "share/field/field_utils.hpp" + +namespace scream { + +class RefiningRemapperP2PTester : public RefiningRemapperP2P { +public: + RefiningRemapperP2PTester (const grid_ptr_type& tgt_grid, + const std::string& map_file) + : RefiningRemapperP2P(tgt_grid,map_file) {} + + ~RefiningRemapperP2PTester () = default; +}; + +Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid) +{ + const auto u = ekat::units::Units::nondimensional(); + const auto CMP = ShortFieldTagsNames::CMP; + const auto& gn = grid.name(); + const auto ndims = 2; + Field f; + switch (lt) { + case LayoutType::Scalar2D: + f = Field(FieldIdentifier(name,grid.get_2d_scalar_layout(),u,gn)); break; + case LayoutType::Vector2D: + f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(CMP,ndims),u,gn)); break; + case LayoutType::Scalar3D: + f = Field(FieldIdentifier(name,grid.get_3d_scalar_layout(true),u,gn)); break; + f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + case LayoutType::Vector3D: + f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(false,CMP,ndims),u,gn)); break; + f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + default: + EKAT_ERROR_MSG ("Invalid layout type for this unit test.\n"); + } + f.allocate_view(); + + return f; +} + +template +Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid, Engine& engine) { + auto f = create_field(name,lt,grid); + + // Use discrete_distribution to get an integer, then use that as exponent for 2^-n. + // This guarantees numbers that are exactly represented as FP numbers, which ensures + // the test will produce the expected answer, regardless of how math ops are performed. + using IPDF = std::discrete_distribution; + IPDF ipdf ({1,1,1,1,1,1,1,1,1,1}); + auto pdf = [&](Engine& e) { + return Real(std::pow(2,ipdf(e))); + }; + randomize(f,engine,pdf); + + return f; +} + +Field all_gather_field (const Field& f, const ekat::Comm& comm) { + constexpr auto COL = ShortFieldTagsNames::COL; + const auto& fid = f.get_header().get_identifier(); + const auto& fl = fid.get_layout(); + int col_size = fl.strip_dim(COL).size(); + auto tags = fl.tags(); + auto dims = fl.dims(); + int my_cols = dims[0];; + comm.all_reduce(&my_cols, &dims.front(), 1, MPI_SUM ); + FieldLayout gfl(tags,dims); + FieldIdentifier gfid("g" + f.name(),gfl,fid.get_units(),fid.get_grid_name(),fid.data_type()); + Field gf(gfid); + gf.allocate_view(); + std::vector data_vec(col_size); + for (int pid=0,offset=0; pid(),icol).data(); + } else { + data = data_vec.data(); + } + break; + case 2: + if (pid==comm.rank()) { + data = ekat::subview(f.get_view(),icol).data(); + } else { + data = data_vec.data(); + } + break; + case 3: + if (pid==comm.rank()) { + data = ekat::subview(f.get_view(),icol).data(); + } else { + data = data_vec.data(); + } + break; + default: + EKAT_ERROR_MSG ( + "Unexpected rank in RefiningRemapperP2P unit test.\n" + " - field name: " + f.name() + "\n"); + } + comm.broadcast(data,col_size,pid); + auto gdata = gf.get_internal_view_data()+offset; + std::copy(data,data+col_size,gdata); + } + } + return gf; +} + +void write_map_file (const std::string& filename, const int ngdofs_src) { + // Add a dof in the middle of two coarse dofs + const int ngdofs_tgt = 2*ngdofs_src-1; + + // Existing dofs are "copied", added dofs are averaged from neighbors + const int nnz = ngdofs_src + 2*(ngdofs_src-1); + + scorpio::register_file(filename, scorpio::FileMode::Write); + + scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); + scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); + scorpio::register_dimension(filename, "n_s", "n_s", nnz, false); + + scorpio::register_variable(filename, "col", "col", "1", {"n_s"}, "int", "int", ""); + scorpio::register_variable(filename, "row", "row", "1", {"n_s"}, "int", "int", ""); + scorpio::register_variable(filename, "S", "S", "1", {"n_s"}, "double", "double", ""); + + std::vector dofs(nnz); + std::iota(dofs.begin(),dofs.end(),0); + scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); + scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); + scorpio::set_dof(filename,"S", dofs.size(),dofs.data()); + + scorpio::eam_pio_enddef(filename); + + std::vector col(nnz), row(nnz); + std::vector S(nnz); + for (int i=0; iget_dofs_gids().get_view(); + for (int i=0; iget_num_local_dofs(); ++i) { + int q = dofs_h[i] / 2; + if (dofs_h[i] % 2 == 0) { + dofs_h[i] = q; + } else { + dofs_h[i] = ngdofs_src + q; + } + } + tgt_grid->get_dofs_gids().sync_to_dev(); + + // Test bad registrations separately, since they corrupt the remapper state for later + { + auto r = std::make_shared(tgt_grid,filename); + auto src_grid = r->get_src_grid(); + r->registration_begins(); + Field bad_src(FieldIdentifier("",src_grid->get_2d_scalar_layout(),ekat::units::m,src_grid->name(),DataType::IntType)); + Field bad_tgt(FieldIdentifier("",tgt_grid->get_2d_scalar_layout(),ekat::units::m,tgt_grid->name(),DataType::IntType)); + CHECK_THROWS (r->register_field(bad_src,bad_tgt)); // not allocated + bad_src.allocate_view(); + bad_tgt.allocate_view(); + CHECK_THROWS (r->register_field(bad_src,bad_tgt)); // bad data type (must be real) + } + + auto r = std::make_shared(tgt_grid,filename); + auto src_grid = r->get_src_grid(); + + auto bundle_src = create_field("bundle3d_src",LayoutType::Vector3D,*src_grid,engine); + auto s2d_src = create_field("s2d_src",LayoutType::Scalar2D,*src_grid,engine); + auto v2d_src = create_field("v2d_src",LayoutType::Vector2D,*src_grid,engine); + auto s3d_src = create_field("s3d_src",LayoutType::Scalar3D,*src_grid,engine); + auto v3d_src = create_field("v3d_src",LayoutType::Vector3D,*src_grid,engine); + + auto bundle_tgt = create_field("bundle3d_tgt",LayoutType::Vector3D,*tgt_grid); + auto s2d_tgt = create_field("s2d_tgt",LayoutType::Scalar2D,*tgt_grid); + auto v2d_tgt = create_field("v2d_tgt",LayoutType::Vector2D,*tgt_grid); + auto s3d_tgt = create_field("s3d_tgt",LayoutType::Scalar3D,*tgt_grid); + auto v3d_tgt = create_field("v3d_tgt",LayoutType::Vector3D,*tgt_grid); + + r->registration_begins(); + r->register_field(s2d_src,s2d_tgt); + r->register_field(v2d_src,v2d_tgt); + r->register_field(s3d_src,s3d_tgt); + r->register_field(v3d_src,v3d_tgt); + r->register_field(bundle_src.get_component(0),bundle_tgt.get_component(0)); + r->register_field(bundle_src.get_component(1),bundle_tgt.get_component(1)); + r->registration_ends(); + + // Run remap + CHECK_THROWS (r->remap(false)); // No backward remap + r->remap(true); + + // Gather global copies (to make checks easier) and check src/tgt fields + auto gs2d_src = all_gather_field(s2d_src,comm); + auto gv2d_src = all_gather_field(v2d_src,comm); + auto gs3d_src = all_gather_field(s3d_src,comm); + auto gv3d_src = all_gather_field(v3d_src,comm); + auto gbundle_src = all_gather_field(bundle_src,comm); + + auto gs2d_tgt = all_gather_field(s2d_tgt,comm); + auto gv2d_tgt = all_gather_field(v2d_tgt,comm); + auto gs3d_tgt = all_gather_field(s3d_tgt,comm); + auto gv3d_tgt = all_gather_field(v3d_tgt,comm); + auto gbundle_tgt = all_gather_field(bundle_tgt,comm); + + Real avg; + // Scalar 2D + { + if (comm.am_i_root()) { + printf(" -> Checking 2d scalars .........\n"); + } + bool ok = true; + gs2d_src.sync_to_host(); + gs2d_tgt.sync_to_host(); + + auto src_v = gs2d_src.get_view(); + auto tgt_v = gs2d_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 2d scalars ......... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Vector 2D + { + if (comm.am_i_root()) { + printf(" -> Checking 2d vectors .........\n"); + } + bool ok = true; + gv2d_src.sync_to_host(); + gv2d_tgt.sync_to_host(); + + auto src_v = gv2d_src.get_view(); + auto tgt_v = gv2d_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 2d vectors ......... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Scalar 3D + { + if (comm.am_i_root()) { + printf(" -> Checking 3d scalars .........\n"); + } + bool ok = true; + gs3d_src.sync_to_host(); + gs3d_tgt.sync_to_host(); + + auto src_v = gs3d_src.get_view(); + auto tgt_v = gs3d_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 3d scalars ......... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Vector 3D + { + if (comm.am_i_root()) { + printf(" -> Checking 3d vectors .........\n"); + } + bool ok = true; + gv3d_src.sync_to_host(); + gv3d_tgt.sync_to_host(); + + auto src_v = gv3d_src.get_view(); + auto tgt_v = gv3d_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 3d vectors ......... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Subfields + { + if (comm.am_i_root()) { + printf(" -> Checking 3d subfields .......\n"); + } + bool ok = true; + gbundle_src.sync_to_host(); + gbundle_tgt.sync_to_host(); + + for (int icmp=0; icmp<2; ++icmp) { + auto sf_src = gbundle_src.get_component(icmp); + auto sf_tgt = gbundle_tgt.get_component(icmp); + + auto src_v = sf_src.get_view(); + auto tgt_v = sf_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 3d subfields ....... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Clean up + r = nullptr; + scorpio::eam_pio_finalize(); +} + +} // namespace scream diff --git a/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp new file mode 100644 index 000000000000..d7cef33b07ce --- /dev/null +++ b/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp @@ -0,0 +1,482 @@ +#include + +#include "share/grid/remap/refining_remapper_rma.hpp" +#include "share/grid/point_grid.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_utils.hpp" +#include "share/field/field_utils.hpp" + +namespace scream { + +class RefiningRemapperRMATester : public RefiningRemapperRMA { +public: + RefiningRemapperRMATester (const grid_ptr_type& tgt_grid, + const std::string& map_file) + : RefiningRemapperRMA(tgt_grid,map_file) {} + + ~RefiningRemapperRMATester () = default; + + void test_internals () { + // Test arrays size + const size_t n = m_num_fields; + REQUIRE (m_src_fields.size()==n); + REQUIRE (m_ov_fields.size()==n); + REQUIRE (m_tgt_fields.size()==n); + REQUIRE (m_col_size.size()==n); + REQUIRE (m_col_stride.size()==n); + REQUIRE (m_col_offset.size()==n); + REQUIRE (m_mpi_win.size()==n); + REQUIRE (m_remote_lids.size()==static_cast(m_ov_coarse_grid->get_num_local_dofs())); + REQUIRE (m_remote_pids.size()==static_cast(m_ov_coarse_grid->get_num_local_dofs())); + + // Test field specs + constexpr auto COL = ShortFieldTagsNames::COL; + for (int i=0; iget_num_local_dofs(); + int ngdofs_src = m_src_grid->get_num_global_dofs(); + REQUIRE (m_row_offsets.extent_int(0)==nldofs_tgt+1); + auto row_offsets_h = cmvdc(m_row_offsets); + auto col_lids_h = cmvdc(m_col_lids); + auto weights_h = cmvdc(m_weights); + auto col_gids_h = m_ov_coarse_grid->get_dofs_gids().get_view(); + + auto row_gids_h = m_tgt_grid->get_dofs_gids().get_view(); + for (int i=0; i=beg); + for (int j=beg; j +Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid, Engine& engine) { + auto f = create_field(name,lt,grid); + + // Use discrete_distribution to get an integer, then use that as exponent for 2^-n. + // This guarantees numbers that are exactly represented as FP numbers, which ensures + // the test will produce the expected answer, regardless of how math ops are performed. + using IPDF = std::discrete_distribution; + IPDF ipdf ({1,1,1,1,1,1,1,1,1,1}); + auto pdf = [&](Engine& e) { + return std::pow(2,ipdf(e)); + }; + randomize(f,engine,pdf); + + return f; +} + +Field all_gather_field (const Field& f, const ekat::Comm& comm) { + constexpr auto COL = ShortFieldTagsNames::COL; + const auto& fid = f.get_header().get_identifier(); + const auto& fl = fid.get_layout(); + int col_size = fl.strip_dim(COL).size(); + auto tags = fl.tags(); + auto dims = fl.dims(); + int my_cols = dims[0];; + comm.all_reduce(&my_cols, &dims.front(), 1, MPI_SUM ); + FieldLayout gfl(tags,dims); + FieldIdentifier gfid("g" + f.name(),gfl,fid.get_units(),fid.get_grid_name(),fid.data_type()); + Field gf(gfid); + gf.allocate_view(); + std::vector data_vec(col_size); + for (int pid=0,offset=0; pid(),icol).data(); + } else { + data = data_vec.data(); + } + break; + case 2: + if (pid==comm.rank()) { + data = ekat::subview(f.get_view(),icol).data(); + } else { + data = data_vec.data(); + } + break; + case 3: + if (pid==comm.rank()) { + data = ekat::subview(f.get_view(),icol).data(); + } else { + data = data_vec.data(); + } + break; + default: + EKAT_ERROR_MSG ( + "Unexpected rank in RefiningRemapperRMA unit test.\n" + " - field name: " + f.name() + "\n"); + } + comm.broadcast(data,col_size,pid); + auto gdata = gf.get_internal_view_data()+offset; + std::copy(data,data+col_size,gdata); + } + } + return gf; +} + +void write_map_file (const std::string& filename, const int ngdofs_src) { + // Add a dof in the middle of two coarse dofs + const int ngdofs_tgt = 2*ngdofs_src-1; + + // Existing dofs are "copied", added dofs are averaged from neighbors + const int nnz = ngdofs_src + 2*(ngdofs_src-1); + + scorpio::register_file(filename, scorpio::FileMode::Write); + + scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); + scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); + scorpio::register_dimension(filename, "n_s", "n_s", nnz, false); + + scorpio::register_variable(filename, "col", "col", "1", {"n_s"}, "int", "int", ""); + scorpio::register_variable(filename, "row", "row", "1", {"n_s"}, "int", "int", ""); + scorpio::register_variable(filename, "S", "S", "1", {"n_s"}, "double", "double", ""); + + std::vector dofs(nnz); + std::iota(dofs.begin(),dofs.end(),0); + scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); + scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); + scorpio::set_dof(filename,"S", dofs.size(),dofs.data()); + + scorpio::eam_pio_enddef(filename); + + std::vector col(nnz), row(nnz); + std::vector S(nnz); + for (int i=0; iget_dofs_gids().get_view(); + for (int i=0; iget_num_local_dofs(); ++i) { + int q = dofs_h[i] / 2; + if (dofs_h[i] % 2 == 0) { + dofs_h[i] = q; + } else { + dofs_h[i] = ngdofs_src + q; + } + } + tgt_grid->get_dofs_gids().sync_to_dev(); + + // Test bad registrations separately, since they corrupt the remapper state for later + { + auto r = std::make_shared(tgt_grid,filename); + auto src_grid = r->get_src_grid(); + r->registration_begins(); + Field bad_src(FieldIdentifier("",src_grid->get_2d_scalar_layout(),ekat::units::m,src_grid->name(),DataType::IntType)); + Field bad_tgt(FieldIdentifier("",tgt_grid->get_2d_scalar_layout(),ekat::units::m,tgt_grid->name(),DataType::IntType)); + CHECK_THROWS (r->register_field(bad_src,bad_tgt)); // not allocated + bad_src.allocate_view(); + bad_tgt.allocate_view(); + CHECK_THROWS (r->register_field(bad_src,bad_tgt)); // bad data type (must be real) + } + + auto r = std::make_shared(tgt_grid,filename); + auto src_grid = r->get_src_grid(); + + auto bundle_src = create_field("bundle3d_src",LayoutType::Vector3D,*src_grid,engine); + auto s2d_src = create_field("s2d_src",LayoutType::Scalar2D,*src_grid,engine); + auto v2d_src = create_field("v2d_src",LayoutType::Vector2D,*src_grid,engine); + auto s3d_src = create_field("s3d_src",LayoutType::Scalar3D,*src_grid,engine); + auto v3d_src = create_field("v3d_src",LayoutType::Vector3D,*src_grid,engine); + + auto bundle_tgt = create_field("bundle3d_tgt",LayoutType::Vector3D,*tgt_grid); + auto s2d_tgt = create_field("s2d_tgt",LayoutType::Scalar2D,*tgt_grid); + auto v2d_tgt = create_field("v2d_tgt",LayoutType::Vector2D,*tgt_grid); + auto s3d_tgt = create_field("s3d_tgt",LayoutType::Scalar3D,*tgt_grid); + auto v3d_tgt = create_field("v3d_tgt",LayoutType::Vector3D,*tgt_grid); + + r->registration_begins(); + r->register_field(s2d_src,s2d_tgt); + r->register_field(v2d_src,v2d_tgt); + r->register_field(s3d_src,s3d_tgt); + r->register_field(v3d_src,v3d_tgt); + r->register_field(bundle_src.get_component(0),bundle_tgt.get_component(0)); + r->register_field(bundle_src.get_component(1),bundle_tgt.get_component(1)); + r->registration_ends(); + + // Test remapper internal state + r->test_internals(); + + // Run remap + CHECK_THROWS (r->remap(false)); // No backward remap + r->remap(true); + + // Gather global copies (to make checks easier) and check src/tgt fields + auto gs2d_src = all_gather_field(s2d_src,comm); + auto gv2d_src = all_gather_field(v2d_src,comm); + auto gs3d_src = all_gather_field(s3d_src,comm); + auto gv3d_src = all_gather_field(v3d_src,comm); + auto gbundle_src = all_gather_field(bundle_src,comm); + + auto gs2d_tgt = all_gather_field(s2d_tgt,comm); + auto gv2d_tgt = all_gather_field(v2d_tgt,comm); + auto gs3d_tgt = all_gather_field(s3d_tgt,comm); + auto gv3d_tgt = all_gather_field(v3d_tgt,comm); + auto gbundle_tgt = all_gather_field(bundle_tgt,comm); + + Real avg; + // Scalar 2D + { + if (comm.am_i_root()) { + printf(" -> Checking 2d scalars .........\n"); + } + bool ok = true; + gs2d_src.sync_to_host(); + gs2d_tgt.sync_to_host(); + + auto src_v = gs2d_src.get_view(); + auto tgt_v = gs2d_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 2d scalars ......... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Vector 2D + { + if (comm.am_i_root()) { + printf(" -> Checking 2d vectors .........\n"); + } + bool ok = true; + gv2d_src.sync_to_host(); + gv2d_tgt.sync_to_host(); + + auto src_v = gv2d_src.get_view(); + auto tgt_v = gv2d_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 2d vectors ......... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Scalar 3D + { + if (comm.am_i_root()) { + printf(" -> Checking 3d scalars .........\n"); + } + bool ok = true; + gs3d_src.sync_to_host(); + gs3d_tgt.sync_to_host(); + + auto src_v = gs3d_src.get_view(); + auto tgt_v = gs3d_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 3d scalars ......... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Vector 3D + { + if (comm.am_i_root()) { + printf(" -> Checking 3d vectors .........\n"); + } + bool ok = true; + gv3d_src.sync_to_host(); + gv3d_tgt.sync_to_host(); + + auto src_v = gv3d_src.get_view(); + auto tgt_v = gv3d_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 3d vectors ......... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Subfields + { + if (comm.am_i_root()) { + printf(" -> Checking 3d subfields .......\n"); + } + bool ok = true; + gbundle_src.sync_to_host(); + gbundle_tgt.sync_to_host(); + + for (int icmp=0; icmp<2; ++icmp) { + auto sf_src = gbundle_src.get_component(icmp); + auto sf_tgt = gbundle_tgt.get_component(icmp); + + auto src_v = sf_src.get_view(); + auto tgt_v = sf_tgt.get_view(); + + // Coarse grid cols are just copied + for (int icol=0; icol Checking 3d subfields ....... %s\n",ok ? "PASS" : "FAIL"); + } + } + + // Clean up + r = nullptr; + scorpio::eam_pio_finalize(); +} + +} // namespace scream diff --git a/components/eamxx/src/share/tests/utils_tests.cpp b/components/eamxx/src/share/tests/utils_tests.cpp index 722de5e57e6e..e8a9c4f5c4ce 100644 --- a/components/eamxx/src/share/tests/utils_tests.cpp +++ b/components/eamxx/src/share/tests/utils_tests.cpp @@ -5,6 +5,7 @@ #include "share/util/scream_utils.hpp" #include "share/util/scream_time_stamp.hpp" #include "share/util/scream_setup_random_test.hpp" +#include "share/scream_config.hpp" TEST_CASE("contiguous_superset") { using namespace scream; @@ -153,9 +154,11 @@ TEST_CASE ("time_stamp") { ts3 += 1; ts4 += 1; #ifdef SCREAM_HAS_LEAP_YEAR + REQUIRE (use_leap_year()); REQUIRE (ts2.get_month()==2); REQUIRE (ts3.get_month()==2); #else + REQUIRE (not use_leap_year()); REQUIRE (ts2.get_month()==3); REQUIRE (ts3.get_month()==3); #endif diff --git a/components/eamxx/src/share/tests/vertical_interp_tests.cpp b/components/eamxx/src/share/tests/vertical_interp_tests.cpp index adbc4daf89ef..c3e5abbfa0b6 100644 --- a/components/eamxx/src/share/tests/vertical_interp_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_interp_tests.cpp @@ -27,6 +27,7 @@ void run(){ //4) n_layers_src= 2*P+1, n_layers_tgt= 2*P-1 //For each scenario target levels are at the midpoint and so should be the average //of the source layers. + // TODO: ASD - Add a test for when the source pressure is a single column and target is multiple columns int n_layers_src[4] = {2*P,2*P+1,2*P,2*P+1}; int n_layers_tgt[4] = {2*P,2*P,2*P-1,2*P-1}; diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index 8848be926f5d..6ec4a40c1236 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -51,10 +51,12 @@ void print (const std::string& msg, const ekat::Comm& comm) { std::shared_ptr build_src_grid(const ekat::Comm& comm, const int nldofs_src, const int nlevs_src) { + using gid_type = AbstractGrid::gid_type; + auto src_grid = std::make_shared("src",nldofs_src,nlevs_src,comm); auto src_dofs = src_grid->get_dofs_gids(); - auto src_dofs_h = src_dofs.get_view(); + auto src_dofs_h = src_dofs.get_view(); std::iota(src_dofs_h.data(),src_dofs_h.data()+nldofs_src,nldofs_src*comm.rank()); src_dofs.sync_to_dev(); @@ -97,9 +99,9 @@ void create_remap_file(const std::string& filename, const int nlevs, const std:: scorpio::register_file(filename, scorpio::FileMode::Write); - scorpio::register_dimension(filename,"nlevs","nlevs",nlevs, false); + scorpio::register_dimension(filename,"lev","lev",nlevs, false); - scorpio::register_variable(filename,"p_levs","p_levs","none",{"nlevs"},"real","real","Real-nlevs"); + scorpio::register_variable(filename,"p_levs","p_levs","none",{"lev"},"real","real","Real-lev"); scorpio::set_dof(filename,"p_levs",dofs_p.size(),dofs_p.data()); @@ -111,6 +113,7 @@ void create_remap_file(const std::string& filename, const int nlevs, const std:: } TEST_CASE ("vertical_remap") { + using gid_type = AbstractGrid::gid_type; // -------------------------------------- // // Init MPI and PIO // @@ -257,7 +260,7 @@ TEST_CASE ("vertical_remap") { // values was, even if that data was off rank. auto pmid_v = pmid_src.get_view(); auto pint_v = pint_src.get_view(); - auto src_gids = remap->get_src_grid()->get_dofs_gids().get_view(); + auto src_gids = remap->get_src_grid()->get_dofs_gids().get_view(); for (const auto& f : src_f) { const auto& l = f.get_header().get_identifier().get_layout(); switch (get_layout_type(l.tags())) { @@ -317,7 +320,7 @@ TEST_CASE ("vertical_remap") { // -------------------------------------- // print (" -> check tgt fields ...\n",comm); - const auto tgt_gids = tgt_grid->get_dofs_gids().get_view(); + const auto tgt_gids = tgt_grid->get_dofs_gids().get_view(); const int ntgt_gids = tgt_gids.size(); for (size_t ifield=0; ifield + +/* + * The atm configuration created in this simple test is fully + * configurable from input yaml file, just like any atm instance. + * Notice that, despite the fact that the source code is the same + * for all atm configurations, in order to use a particular atm + * proc (e.g., a physics package), this source code must be linked + * against the lib providing such atm proc, otherwise it won't + * get registered in the atm proc factory. + */ + +TEST_CASE("scream_ad_test") { + using namespace scream; + using namespace scream::control; + + // Create a comm + ekat::Comm atm_comm (MPI_COMM_WORLD); + + // User can prescribe input file name via --ekat-test-params ifile= + auto& session = ekat::TestSession::get(); + session.params.emplace("ifile","input.yaml"); + std::string fname = session.params["ifile"]; + + // Load ad parameter list + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + ad_params.print(); + + // Time stepping parameters + auto& ts = ad_params.sublist("time_stepping"); + const auto dt = ts.get("time_step"); + const auto nsteps = ts.get("number_of_steps"); + const auto run_t0_str = ts.get("run_t0"); + const auto run_t0 = util::str_to_time_stamp(run_t0_str); + const auto case_t0_str = ts.get("case_t0",run_t0_str); + const auto case_t0 = util::str_to_time_stamp(case_t0_str); + + // Register all atm procs, grids manager, and diagnostics in the respective factories + register_dynamics(); + register_physics(); + register_diagnostics(); + register_mesh_free_grids_manager(); + + // Create the driver + AtmosphereDriver ad; + + // Init, run, and finalize + ad.initialize(atm_comm,ad_params,run_t0,case_t0); + + if (atm_comm.am_i_root()) { + printf("Start time stepping loop... [ 0%%]\n"); + } + for (int i=0; i remapper; + std::vector active_gases; // other than h2o + +public: + + static TraceGasesWorkaround& singleton() { + static TraceGasesWorkaround self; + return self; + } + + void set_restart (const bool is_restart) { + restart = is_restart; + } + void set_remapper (const std::shared_ptr& remap_ptr) { + remapper = remap_ptr; + } + void erase_remapper () { remapper = nullptr; } + void add_active_gas (const std::string gas_name) { + active_gases.push_back(gas_name); + } + + bool is_restart() const { return restart; } + std::shared_ptr get_remapper() const { return remapper; } + std::vector get_active_gases() const { return active_gases; } +}; + +extern bool fvphyshack; +void fv_phys_rrtmgp_active_gases_set_restart(const bool restart); + +} // namespace scream diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp new file mode 100644 index 000000000000..a3d132bf4f82 --- /dev/null +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -0,0 +1,388 @@ +#include "share/util/eamxx_time_interpolation.hpp" + +namespace scream{ +namespace util { + +/*-----------------------------------------------------------------------------------------------*/ +// Constructors +TimeInterpolation::TimeInterpolation( + const grid_ptr_type& grid +) +{ + // Given the grid initialize field managers to store interpolation data + m_fm_time0 = std::make_shared(grid); + m_fm_time1 = std::make_shared(grid); + m_fm_time0->registration_begins(); + m_fm_time0->registration_ends(); + m_fm_time1->registration_begins(); + m_fm_time1->registration_ends(); +} +/*-----------------------------------------------------------------------------------------------*/ +TimeInterpolation::TimeInterpolation( + const grid_ptr_type& grid, + const vos_type& list_of_files +) : TimeInterpolation(grid) +{ + set_file_data_triplets(list_of_files); + m_is_data_from_file = true; +} +/*-----------------------------------------------------------------------------------------------*/ +void TimeInterpolation::finalize() +{ + if (m_is_data_from_file) { + m_file_data_atm_input.finalize(); + m_is_data_from_file=false; + } +} +/*-----------------------------------------------------------------------------------------------*/ +/* A function to perform time interpolation using data from all the fields stored in the local + * field managers. + * Conducts a simple linear interpolation between two points using + * y* = w*y0 + (1-w)*y1 + * where w = (t1-t*)/(t1-t0) + * Input: + * time_in - A timestamp to interpolate onto. + * Output + * A map of (field name) and (Field) pairs such that each field is the interpolated values and + * the map makes it possible to quickly query individual outputs. + */ +void TimeInterpolation::perform_time_interpolation(const TimeStamp& time_in) +{ + // Declare the output map + std::map interpolated_fields; + + // If data is handled by files we need to check that the timestamps are still relevant + if (m_file_data_triplets.size()>0) { + check_and_update_data(time_in); + } + + // Gather weights for interpolation. Note, timestamp differences are integers and we need a + // real defined weight. + const Real w_num = m_time1 - time_in; + const Real w_den = m_time1 - m_time0; + const Real weight0 = w_num/w_den; + const Real weight1 = 1.0-weight0; + + // Cycle through all stored fields and conduct the time interpolation + for (auto name : m_field_names) + { + const auto& field0 = m_fm_time0->get_field(name); + const auto& field1 = m_fm_time1->get_field(name); + auto field_out = m_interp_fields.at(name); + field_out.deep_copy(field0); + field_out.update(field1,weight1,weight0); + } +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function which registers a field in the local field managers. + * Input: + * field_in - Is a field with the appropriate dimensions and metadata to match the interpolation + * Output: + * None + */ +void TimeInterpolation::add_field(const Field& field_in, const bool store_shallow_copy) +{ + // First check that we haven't already added a field with the same name. + const std::string name = field_in.name(); + EKAT_REQUIRE_MSG(!m_fm_time0->has_field(name) and !m_fm_time1->has_field(name), + "Error!! TimeInterpolation:add_field, field + " << name << " has already been added." << "\n"); + + // Clone the field for each field manager to get all the metadata correct. + auto field0 = field_in.clone(); + auto field1 = field_in.clone(); + m_fm_time0->add_field(field0); + m_fm_time1->add_field(field1); + if (store_shallow_copy) { + // Then we want to store the actual field_in and override it when interpolating + m_interp_fields.emplace(name,field_in); + } else { + // We want to store a copy of the field but not ovveride + auto field_out = field_in.clone(); + m_interp_fields.emplace(name,field_out); + } + m_field_names.push_back(name); +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function to shift all data from time1 to time0, update timestamp for time0 + */ +void TimeInterpolation::shift_data() +{ + for (auto name : m_field_names) + { + auto& field0 = m_fm_time0->get_field(name); + auto& field1 = m_fm_time1->get_field(name); + std::swap(field0,field1); + } + m_file_data_atm_input.set_field_manager(m_fm_time1); +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function which will initialize the TimeStamps. + * Input: + * ts_in - A timestamp to set both time0 and time1 to. + * + * At initialization we assume that only the first timestep of data has been set. Subsequent + * timesteps of data are added using update_data and then a call to update_timestamp will + * shift time0 to time1 and update time1. + */ +void TimeInterpolation::initialize_timestamps(const TimeStamp& ts_in) +{ + m_time0 = ts_in; + m_time1 = ts_in; +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function which will initialize field data given an input field. + * Input: + * field_in - A field with a name matching one of the fields in the interpolator. Data will be + * shifted and copied. + */ +void TimeInterpolation::initialize_data_from_field(const Field& field_in) +{ + const auto name = field_in.name(); + auto field0 = m_fm_time0->get_field(name); + auto field1 = m_fm_time1->get_field(name); + field0.deep_copy(field_in); + field1.deep_copy(field_in); + auto ts = field_in.get_header().get_tracking().get_time_stamp(); + m_time0 = ts; + m_time1 = ts; +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function which will initialize the full set of fields given a list of files pointing to where + * the data can be found. + * Input: + * list_of_files: A vector of strings representing all the files where interpolation data can be + * gathered. + */ +void TimeInterpolation::initialize_data_from_files() +{ + auto triplet_curr = m_file_data_triplets[m_triplet_idx]; + // Initialize the AtmosphereInput object that will be used to gather data + ekat::ParameterList input_params; + input_params.set("Field Names",m_field_names); + input_params.set("Filename",triplet_curr.filename); + m_file_data_atm_input = AtmosphereInput(input_params,m_fm_time1); + // Assign the mask value gathered from the FillValue found in the source file. + // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? + float var_fill_value; + for (auto& name : m_field_names) { + scorpio::get_variable_metadata(triplet_curr.filename,name,"_FillValue",var_fill_value); + auto& field0 = m_fm_time0->get_field(name); + field0.get_header().set_extra_data("mask_value",var_fill_value); + auto& field1 = m_fm_time1->get_field(name); + field1.get_header().set_extra_data("mask_value",var_fill_value); + auto& field_out = m_interp_fields.at(name); + field_out.get_header().set_extra_data("mask_value",var_fill_value); + } + // Read first snap of data and shift to time0 + read_data(); + shift_data(); + update_timestamp(triplet_curr.timestamp); + // Advance the iterator and read the next set of data for time1 + ++m_triplet_idx; + read_data(); +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function which will update the timestamps by shifting time1 to time0 and setting time1. + * Input: + * ts_in - A timestamp for the most recent timestamp of the interpolation data. + * + * It is assumed that any updated time is meant to replace time1 and that the time1 + * should be shifted to time0 + */ +void TimeInterpolation::update_timestamp(const TimeStamp& ts_in) +{ + m_time0 = m_time1; + m_time1 = ts_in; +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function which will update field data given an input field. Useful for time interpolation + * related to the EAMxx state. + * Input: + * field_in - A field with a name matching one of the fields in the interpolator. Data will be + * shifted and copied. + * + * It is assumed that any updated data is meant to replace the data at time1 and that the time1 + * data should be shifted to time0 + */ +void TimeInterpolation::update_data_from_field(const Field& field_in) +{ + const auto name = field_in.name(); + auto& field0 = m_fm_time0->get_field(name); + auto& field1 = m_fm_time1->get_field(name); + std::swap(field0,field1); + // Now that we have swapped field0 and field 1 we need to grab field 1 from the field manager again. + // Alternatively we could just update `field0` which is now inside m_fm_time1, but choosing this + // approach for code readability. + auto& field1_new = m_fm_time1->get_field(name); + field1_new.deep_copy(field_in); +} +/*-----------------------------------------------------------------------------------------------*/ +void TimeInterpolation::print() +{ + printf("Settings for time interpolator...\n"); + printf("Time 0 = %s\n",m_time0.to_string().c_str()); + printf("Time 1 = %s\n",m_time1.to_string().c_str()); + printf("List of Fields in interpolator:\n"); + for (auto name : m_field_names) + { + printf(" - %16s\n",name.c_str()); + } + +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function to organize the data available from data files by time. This is necessary to quickly + * grab the appropriate data each time interpolation is requested. + * Input: + * list_of_files - Is a vector of strings representing all files that can be used for data. + * + * We create a reference timestamp to use when sorting the data snaps. + */ +void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { + EKAT_REQUIRE_MSG(list_of_files.size()>0,"ERROR! TimeInterpolation::set_file_data_triplets - the list of files is empty. Please check."); + TimeStamp ts_ref; + // The first step is to grab all relevant metadata for the DataFromFileTriplet objects. + // We will store the times in a map and take advantage of maps natural sorting to organize the triplets + // in chronological order. This ensures that if the list of files does not represent the chronological + // order to the data we will still have sorted data. + vos_type filenames_tmp; + std::vector timestamps_tmp; + std::vector time_idx_tmp; + std::map map_of_times_to_vector_idx; + int running_idx = 0; + for (size_t ii=0; ii(time_units_tmp); + int time_mult; + if (time_units.find("seconds") != std::string::npos) { + time_mult = 1; + } else if (time_units.find("minutes") != std::string::npos) { + time_mult = 60; + } else if (time_units.find("hours") != std::string::npos) { + time_mult = 3600; + } else if (time_units.find("days") != std::string::npos) { + time_mult = 86400; + } else { + EKAT_ERROR_MSG("Error!! TimeInterpolation::set_file_triplets - unsupported units of time = (" << time_units << ") in source data file " << filename << ", supported units are: seconds, minutes, hours and days"); + } + // Gather information about time in this file + if (ii==0) { + ts_ref = ts_file_start; + } + scorpio::register_file(filename,scorpio::Read); + const int ntime = scorpio::get_dimlen(filename,"time"); + for (int tt=0; tt0) { + ts_snap += (time_snap*time_mult); + } + auto time = ts_snap.seconds_from(ts_ref); + // Sanity check that we don't have multiples of the same timesnap + EKAT_REQUIRE_MSG(map_of_times_to_vector_idx.count(time)==0,"Error! TimeInterpolation::set_file_data_triplets - The same time step has been encountered more than once in the data files, please check\n" + << " TimeStamp: " << ts_snap.to_string() << "\n" + << " Filename: " << filename << "\n"); + map_of_times_to_vector_idx.emplace(time,running_idx); + filenames_tmp.push_back(filename); + timestamps_tmp.push_back(ts_snap); + time_idx_tmp.push_back(tt); + ++running_idx; + } + } + // Now that we have gathered all of the timesnaps we can arrange them in order as DataFromFileTriplet objects. + // Taking advantage of maps automatically self-sorting by the first arg. + for (auto a : map_of_times_to_vector_idx) { + auto idx = a.second; + DataFromFileTriplet my_trip; + my_trip.filename = filenames_tmp[idx]; + my_trip.timestamp = timestamps_tmp[idx]; + my_trip.time_idx = time_idx_tmp[idx]; + m_file_data_triplets.push_back(my_trip); + } + // Finally set the iterator to point to the first triplet. + m_triplet_idx = 0; +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function to read a new set of data from file using the current iterator pointing to the current + * DataFromFileTriplet. + */ +void TimeInterpolation::read_data() +{ + const auto triplet_curr = m_file_data_triplets[m_triplet_idx]; + if (triplet_curr.filename != m_file_data_atm_input.get_filename()) { + // Then we need to close this input stream and open a new one + m_file_data_atm_input.finalize(); + ekat::ParameterList input_params; + input_params.set("Field Names",m_field_names); + input_params.set("Filename",triplet_curr.filename); + m_file_data_atm_input = AtmosphereInput(input_params,m_fm_time1); + // Also determine the FillValue, if used + // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? + float var_fill_value; + for (auto& name : m_field_names) { + scorpio::get_variable_metadata(triplet_curr.filename,name,"_FillValue",var_fill_value); + auto& field = m_fm_time1->get_field(name); + field.get_header().set_extra_data("mask_value",var_fill_value); + } + } + m_file_data_atm_input.read_variables(triplet_curr.time_idx); + m_time1 = triplet_curr.timestamp; +} +/*-----------------------------------------------------------------------------------------------*/ +/* Function to check the current set of interpolation data against a timestamp and, if needed, + * update the set of interpolation data to ensure the passed timestamp is within the bounds of + * the interpolation data. + * Input: + * ts_in - A timestamp that we intend to interpolate onto. + */ +void TimeInterpolation::check_and_update_data(const TimeStamp& ts_in) +{ + // First check if the passed timestamp is within the bounds of time0 and time1. + EKAT_REQUIRE_MSG(ts_in.seconds_from(m_time0) >= 0, "ERROR!!! TimeInterpolation::check_and_update_data - " + << "Current timestamp of " << ts_in.to_string() << " is lower than the TimeInterpolation bounds of " << m_time0.to_string()); + if (m_time1.seconds_from(ts_in) < 0) { + // The timestamp is out of bounds, need to load new data. + // First cycle through the DataFromFileTriplet's to find a timestamp that is greater than this one. + bool found = false; + int step_cnt = 0; // Track how many triplets we passed to find one that worked. + while (m_triplet_idx < static_cast(m_file_data_triplets.size())) { + ++m_triplet_idx; + ++step_cnt; + auto ts_tmp = m_file_data_triplets[m_triplet_idx].timestamp; + if (ts_tmp.seconds_from(ts_in) >= 0) { + // This timestamp is greater than the input timestamp, we can use it + found = true; + break; + } + } + EKAT_REQUIRE_MSG(found,"ERROR!! TimeInterpolation::check_and_update_data - timestamp " << ts_in.to_string() << "is outside the bounds of the set of data files." << "\n" + << " TimeStamp time0: " << m_time0.to_string() << "\n" + << " TimeStamp time1: " << m_time1.to_string() << "\n"); + // Now we need to make sure we didn't jump more than one triplet, if we did then the data at time0 is + // incorrect. + if (step_cnt>1) { + // Then we need to populate data for time1 as the previous triplet before shifting data to time0 + --m_triplet_idx; + read_data(); + ++m_triplet_idx; + } + // We shift the time1 data to time0 and read the new data. + shift_data(); + update_timestamp(m_file_data_triplets[m_triplet_idx].timestamp); + read_data(); + // Sanity Check + bool current_data_check = (ts_in.seconds_from(m_time0) >= 0) and (m_time1.seconds_from(ts_in) >= 0); + EKAT_REQUIRE_MSG(current_data_check,"ERROR!! TimeInterpolation::check_and_update_data - Something went wrong in updating data:\n" + << " TimeStamp IN: " << ts_in.to_string() << "\n" + << " TimeStamp time0: " << m_time0.to_string() << "\n" + << " TimeStamp time1: " << m_time1.to_string() << "\n"); + + } +} +/*-----------------------------------------------------------------------------------------------*/ + +} // namespace util +} // namespace scream diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.hpp b/components/eamxx/src/share/util/eamxx_time_interpolation.hpp new file mode 100644 index 000000000000..89d4c917d276 --- /dev/null +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.hpp @@ -0,0 +1,94 @@ +#ifndef EAMXX_TIME_INTERPOLATION_HPP +#define EAMXX_TIME_INTERPOLATION_HPP + +#include "share/grid/abstract_grid.hpp" + +#include "share/util/scream_time_stamp.hpp" + +#include "share/field/field.hpp" +#include "share/field/field_manager.hpp" + +#include "share/io/scorpio_input.hpp" + +namespace scream{ +namespace util { + +class TimeInterpolation { +public: + using grid_ptr_type = std::shared_ptr; + using vos_type = std::vector; + using fm_type = std::shared_ptr; + + // Constructors & Destructor + TimeInterpolation() = default; + TimeInterpolation(const grid_ptr_type& grid); + TimeInterpolation(const grid_ptr_type& grid, const vos_type& list_of_files); + ~TimeInterpolation () = default; + + // Running the interpolation + void initialize_timestamps(const TimeStamp& ts_in); + void initialize_data_from_field(const Field& field_in); + void initialize_data_from_files(); + void update_data_from_field(const Field& field_in); + void update_timestamp(const TimeStamp& ts_in); + void perform_time_interpolation(const TimeStamp& time_in); + void finalize(); + + // Build interpolator + void add_field(const Field& field_in, const bool store_shallow_copy=false); + + // Getters + Field get_field(const std::string& name) { + return m_interp_fields.at(name); + }; + + // Informational + void print(); + +protected: + + // Internal structure to store data source triplets (when using data from file) + // For each timesnap of data we have access to this triplet stores the + // - filename + // - timestamp + // - time index in the file. + // Note, in many cases we will have files with multiple snaps of data and we + // need a good way to organize this information. + struct DataFromFileTriplet { + public: + std::string filename; + TimeStamp timestamp; + int time_idx; + }; + + // Helper functions to shift data + void shift_data(); + + // For the case where forcing data comes from files + void set_file_data_triplets(const vos_type& list_of_files); + void read_data(); + void check_and_update_data(const TimeStamp& ts_in); + + // Local field managers used to store two time snaps of data for interpolation + fm_type m_fm_time0; + fm_type m_fm_time1; + vos_type m_field_names; + std::map m_interp_fields; + + // Store the timestamps associated with the two time snaps + TimeStamp m_time0; + TimeStamp m_time1; + + // Variables related to the case where we use data from file + std::vector m_file_data_triplets; + int m_triplet_idx; + AtmosphereInput m_file_data_atm_input; + bool m_is_data_from_file=false; + + +}; // class TimeInterpolation + +} // namespace util +} // namespace scream + +#endif // EAMXX_TIME_INTERPOLATION_HPP diff --git a/components/eamxx/src/share/util/scream_bfbhash.cpp b/components/eamxx/src/share/util/scream_bfbhash.cpp new file mode 100644 index 000000000000..ea2f10a68455 --- /dev/null +++ b/components/eamxx/src/share/util/scream_bfbhash.cpp @@ -0,0 +1,25 @@ +#include "share/util/scream_bfbhash.hpp" + +namespace scream { +namespace bfbhash { + +static void reduce_hash (void* invec, void* inoutvec, int* len, MPI_Datatype* /* datatype */) { + const int n = *len; + const auto* s = reinterpret_cast(invec); + auto* d = reinterpret_cast(inoutvec); + for (int i = 0; i < n; ++i) hash(s[i], d[i]); +} + +int all_reduce_HashType (MPI_Comm comm, const HashType* sendbuf, HashType* rcvbuf, + int count) { + static_assert(sizeof(long long int) == sizeof(HashType), + "HashType must have size sizeof(long long int)."); + MPI_Op op; + MPI_Op_create(reduce_hash, true, &op); + const auto stat = MPI_Allreduce(sendbuf, rcvbuf, count, MPI_LONG_LONG_INT, op, comm); + MPI_Op_free(&op); + return stat; +} + +} // namespace bfbhash +} // namespace scream diff --git a/components/eamxx/src/share/util/scream_bfbhash.hpp b/components/eamxx/src/share/util/scream_bfbhash.hpp new file mode 100644 index 000000000000..475f4dfa8810 --- /dev/null +++ b/components/eamxx/src/share/util/scream_bfbhash.hpp @@ -0,0 +1,56 @@ +#ifndef SCREAM_BFBHASH_HPP +#define SCREAM_BFBHASH_HPP + +#include + +#include +#include + +namespace scream { +namespace bfbhash { + +typedef std::uint64_t HashType; + +KOKKOS_INLINE_FUNCTION void hash (const HashType v, HashType& accum) { + constexpr auto first_bit = 1ULL << 63; + accum += ~first_bit & v; // no overflow + accum ^= first_bit & v; // handle most significant bit +} + +KOKKOS_INLINE_FUNCTION void hash (const double v_, HashType& accum) { + static_assert(sizeof(double) == sizeof(HashType), + "HashType must have size sizeof(double)."); + HashType v; + std::memcpy(&v, &v_, sizeof(HashType)); + hash(v, accum); +} + +KOKKOS_INLINE_FUNCTION void hash (const float v, HashType& accum) { + hash(double(v), accum); +} + +// For Kokkos::parallel_reduce. +template +struct HashReducer { + typedef HashReducer reducer; + typedef HashType value_type; + typedef Kokkos::View result_view_type; + + KOKKOS_INLINE_FUNCTION HashReducer (value_type& value_) : value(value_) {} + KOKKOS_INLINE_FUNCTION void join (value_type& dest, const value_type& src) const { hash(src, dest); } + KOKKOS_INLINE_FUNCTION void init (value_type& val) const { val = 0; } + KOKKOS_INLINE_FUNCTION value_type& reference () const { return value; } + KOKKOS_INLINE_FUNCTION bool references_scalar () const { return true; } + KOKKOS_INLINE_FUNCTION result_view_type view () const { return result_view_type(&value, 1); } + +private: + value_type& value; +}; + +int all_reduce_HashType(MPI_Comm comm, const HashType* sendbuf, HashType* rcvbuf, + int count); + +} // namespace bfbhash +} // namespace scream + +#endif diff --git a/components/eamxx/src/share/util/scream_combine_ops.hpp b/components/eamxx/src/share/util/scream_combine_ops.hpp index 85f879b2c769..5ac3adfd90dd 100644 --- a/components/eamxx/src/share/util/scream_combine_ops.hpp +++ b/components/eamxx/src/share/util/scream_combine_ops.hpp @@ -1,6 +1,8 @@ #ifndef SCREAM_COMBINE_OPS_HPP #define SCREAM_COMBINE_OPS_HPP +#include "share/util/scream_universal_constants.hpp" + // For KOKKOS_INLINE_FUNCTION #include #include @@ -64,7 +66,7 @@ template::scalar_type> +KOKKOS_FORCEINLINE_FUNCTION +void combine_and_fill (const ScalarIn& newVal, ScalarOut& result, const ScalarOut fill_val, + const CoeffType alpha = CoeffType(1), + const CoeffType beta = CoeffType(0)) +{ + switch (CM) { + case CombineMode::Replace: + combine(newVal,result,alpha,beta); + break; + case CombineMode::Rescale: + if (result != fill_val) { + combine(newVal,result,alpha,beta); + } + break; + case CombineMode::ScaleReplace: + if (newVal == fill_val) { + result = fill_val; + } else { + combine(newVal,result,alpha,beta); + } + break; + case CombineMode::Update: + case CombineMode::ScaleUpdate: + case CombineMode::ScaleAdd: + case CombineMode::Add: + case CombineMode::Multiply: + case CombineMode::Divide: + if (result == fill_val || newVal == fill_val) { + result = fill_val; + } else { + combine(newVal,result,alpha,beta); + } + break; + } +} } // namespace scream diff --git a/components/eamxx/src/share/util/scream_common_physics_functions.hpp b/components/eamxx/src/share/util/scream_common_physics_functions.hpp index 4af6ca969023..92050bccee8a 100644 --- a/components/eamxx/src/share/util/scream_common_physics_functions.hpp +++ b/components/eamxx/src/share/util/scream_common_physics_functions.hpp @@ -51,6 +51,18 @@ struct PhysicsFunctions KOKKOS_INLINE_FUNCTION static ScalarT calculate_density(const ScalarT& pseudo_density, const ScalarT& dz); + //-----------------------------------------------------------------------------------------------// + // Determines the vertical wind velocity given the vertical pressure velocity + // w = - omega / ( density * g ) + // where, + // rho is the vertical wind velocity, [m/s] + // omega is the vertical pressure velocity , [Pa/s] + // g is the gravitational constant, [m/s2] - defined in physics_constants.hpp + //-----------------------------------------------------------------------------------------------// + template + KOKKOS_INLINE_FUNCTION + static ScalarT calculate_vertical_velocity(const ScalarT& omega, const ScalarT& density); + //-----------------------------------------------------------------------------------------------// // Applies Exners Function which follows: // Exner = (P/P0)^(Rd/Cp), @@ -180,6 +192,37 @@ struct PhysicsFunctions KOKKOS_INLINE_FUNCTION static ScalarT calculate_wetmmr_from_drymmr(const ScalarT& drymmr, const ScalarT& qv_dry); + //-----------------------------------------------------------------------------------------------// + // Computes drymmr (mass of a constituent divided by mass of dry air) + // for any wetmmr constituent (mass of a constituent divided by mass of wet air; + // commonly known as mixing ratio) using dry and wet pseudodensities + // drymmr = wetmmr * pdel / pdeldry + // where + // wetmmr is the wet mass mixing ratio of a species + // pseudo_density is wet pseudodensity (pdel) + // pseudo_density_dry is dry pseudodensity (pdeldry) + //-----------------------------------------------------------------------------------------------// + + template + KOKKOS_INLINE_FUNCTION + static ScalarT calculate_drymmr_from_wetmmr_dp_based(const ScalarT& wetmmr, + const ScalarT& pseudo_density, const ScalarT& pseudo_density_dry); + + //-----------------------------------------------------------------------------------------------// + // Computes wetmmr (mass of a constituent divided by mass of wet air) + // for any drymmr constituent (mass of a constituent divided by mass of dry air; + // commonly known as mixing ratio) using dry and wet pseudodensities + // wetmmr = drymmr * pdeldry / pdelwet + // where + // drymmr is the dry mass mixing ratio of a species + // pseudo_density is wet pseudodensity (pdel) + // pseudo_density_dry is dry pseudodensity (pdeldry) + //-----------------------------------------------------------------------------------------------// + template + KOKKOS_INLINE_FUNCTION + static ScalarT calculate_wetmmr_from_drymmr_dp_based(const ScalarT& drymmr, + const ScalarT& pseudo_density, const ScalarT& pseudo_density_dry); + //-----------------------------------------------------------------------------------------------// // Determines the vertical layer thickness using the equation of state: // dz = - (-pseudo_density)*Rd*T_virtual / (p_mid*g) @@ -329,6 +372,13 @@ struct PhysicsFunctions const InputProviderZ& dz, const view_1d& density); + template + KOKKOS_INLINE_FUNCTION + static void calculate_vertical_velocity (const MemberType& team, + const InputProviderOmega& omega, + const InputProviderRho& rho, + const view_1d& w); + template KOKKOS_INLINE_FUNCTION static void exner_function (const MemberType& team, @@ -394,6 +444,22 @@ struct PhysicsFunctions const InputProviderQ& qv_wet, const view_1d& drymmr); + template + KOKKOS_INLINE_FUNCTION + static void calculate_wetmmr_from_drymmr_dp_based (const MemberType& team, + const InputProviderX& drymmr, + const InputProviderPD& pseudo_density, + const InputProviderPD& pseudo_density_dry, + const view_1d& wetmmr); + + template + KOKKOS_INLINE_FUNCTION + static void calculate_drymmr_from_wetmmr_dp_based (const MemberType& team, + const InputProviderX& wetmmr, + const InputProviderPD& pseudo_density, + const InputProviderPD& pseudo_density_dry, + const view_1d& drymmr); + template::calculate_density(const MemberType& team, }); } +template +template +KOKKOS_INLINE_FUNCTION +ScalarT PhysicsFunctions::calculate_vertical_velocity(const ScalarT& omega, const ScalarT& density) +{ + using C = scream::physics::Constants; + + static constexpr auto g = C::gravit; + + return -omega/(density * g); +} + +template +template +KOKKOS_INLINE_FUNCTION +void PhysicsFunctions::calculate_vertical_velocity(const MemberType& team, + const InputProviderOmega& omega, + const InputProviderRho& rho, + const view_1d& w) +{ + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, w.extent(0)), + [&] (const int k) { + w(k) = calculate_vertical_velocity(omega(k), rho(k)); + }); +} + template template KOKKOS_INLINE_FUNCTION @@ -269,6 +295,31 @@ void PhysicsFunctions::calculate_wetmmr_from_drymmr(const MemberType& t }); } +template +template +KOKKOS_INLINE_FUNCTION +ScalarT PhysicsFunctions:: +calculate_wetmmr_from_drymmr_dp_based(const ScalarT& drymmr, + const ScalarT& pseudo_density, const ScalarT& pseudo_density_dry) +{ + return drymmr*pseudo_density_dry/pseudo_density; +} + +template +template +KOKKOS_INLINE_FUNCTION +void PhysicsFunctions::calculate_wetmmr_from_drymmr_dp_based(const MemberType& team, + const InputProviderX& drymmr, + const InputProviderPD& pseudo_density, + const InputProviderPD& pseudo_density_dry, + const view_1d& wetmmr) +{ + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,wetmmr.extent(0)), + [&] (const int k) { + wetmmr(k) = calculate_wetmmr_from_drymmr_dp_based(drymmr(k),pseudo_density(k),pseudo_density_dry(k)); + }); +} + template template KOKKOS_INLINE_FUNCTION @@ -292,6 +343,31 @@ void PhysicsFunctions::calculate_drymmr_from_wetmmr(const MemberType& t }); } +template +template +KOKKOS_INLINE_FUNCTION +ScalarT PhysicsFunctions:: +calculate_drymmr_from_wetmmr_dp_based(const ScalarT& wetmmr, + const ScalarT& pseudo_density, const ScalarT& pseudo_density_dry) +{ + return wetmmr*pseudo_density/pseudo_density_dry; +} + +template +template +KOKKOS_INLINE_FUNCTION +void PhysicsFunctions::calculate_drymmr_from_wetmmr_dp_based(const MemberType& team, + const InputProviderX& wetmmr, + const InputProviderPD& pseudo_density, + const InputProviderPD& pseudo_density_dry, + const view_1d& drymmr) +{ + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,drymmr.extent(0)), + [&] (const int k) { + drymmr(k) = calculate_drymmr_from_wetmmr_dp_based(wetmmr(k),pseudo_density(k),pseudo_density_dry(k)); + }); +} + template template KOKKOS_INLINE_FUNCTION diff --git a/components/eamxx/src/share/util/scream_data_type.hpp b/components/eamxx/src/share/util/scream_data_type.hpp index a292c7c5b651..31ce6b9f8ef7 100644 --- a/components/eamxx/src/share/util/scream_data_type.hpp +++ b/components/eamxx/src/share/util/scream_data_type.hpp @@ -48,6 +48,7 @@ inline std::string e2str (const DataType data_type) { case DataType::IntType: return "int"; case DataType::FloatType: return "float"; case DataType::DoubleType: return "double"; + case DataType::Invalid: return "invalid"; default: EKAT_ERROR_MSG("Error! Unsupported DataType value.\n"); } diff --git a/components/eamxx/src/share/util/scream_time_stamp.cpp b/components/eamxx/src/share/util/scream_time_stamp.cpp index d11cb98f5b8c..fd0ef3785f10 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.cpp +++ b/components/eamxx/src/share/util/scream_time_stamp.cpp @@ -26,21 +26,18 @@ int days_in_month (const int yy, const int mm) { } bool is_leap_year (const int yy) { -#ifdef SCREAM_HAS_LEAP_YEAR - if (yy%4==0) { - // Year is divisible by 4 (minimum requirement) - if (yy%100 != 0) { - // Not a centennial year => leap. - return true; - } else if ((yy/100)%4==0) { - // Centennial year, AND first 2 digids divisible by 4 => leap - return true; + if (use_leap_year()) { + if (yy%4==0) { + // Year is divisible by 4 (minimum requirement) + if (yy%100 != 0) { + // Not a centennial year => leap. + return true; + } else if ((yy/100)%4==0) { + // Centennial year, AND first 2 digids divisible by 4 => leap + return true; + } } } -#else - (void)yy; -#endif - // Either leap year not enabled, or not a leap year at all return false; } @@ -148,7 +145,7 @@ TimeStamp& TimeStamp::operator+=(const double seconds) { // Sanity checks // Note: (x-int(x)) only works for x small enough that can be stored in an int, // but that should be the case here, for use cases in EAMxx. - EKAT_REQUIRE_MSG (seconds>0, "Error! Time must move forward.\n"); + EKAT_REQUIRE_MSG (seconds>=0, "Error! Cannot rewind time.\n"); EKAT_REQUIRE_MSG ((seconds-round(seconds))::epsilon()*10, "Error! Cannot update TimeStamp with non-integral number of seconds " << seconds << "\n"); @@ -202,6 +199,10 @@ TimeStamp& TimeStamp::operator+=(const double seconds) { return *this; } +TimeStamp TimeStamp::clone (const int num_steps) { + return TimeStamp (get_date(),get_time(),num_steps>=0 ? num_steps : m_num_steps); +} + bool operator== (const TimeStamp& ts1, const TimeStamp& ts2) { return ts1.get_date()==ts2.get_date() && ts1.get_time()==ts2.get_time(); } diff --git a/components/eamxx/src/share/util/scream_time_stamp.hpp b/components/eamxx/src/share/util/scream_time_stamp.hpp index e38914f9b770..fdfec5bb0ebf 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.hpp +++ b/components/eamxx/src/share/util/scream_time_stamp.hpp @@ -55,6 +55,9 @@ class TimeStamp { // This method checks that time shifts forward (i.e. that seconds is positive) TimeStamp& operator+= (const double seconds); + // Clones the stamps and sets num steps to given value. If -1, clones num steps too + TimeStamp clone (const int num_steps); + protected: std::vector m_date; // [year, month, day] diff --git a/components/eamxx/src/share/util/scream_universal_constants.hpp b/components/eamxx/src/share/util/scream_universal_constants.hpp index b9f2e694a792..c133cd93cd39 100644 --- a/components/eamxx/src/share/util/scream_universal_constants.hpp +++ b/components/eamxx/src/share/util/scream_universal_constants.hpp @@ -1,6 +1,8 @@ #ifndef SCREAM_UNIVERSAL_CONSTANTS_HPP #define SCREAM_UNIVERSAL_CONSTANTS_HPP +#include + namespace scream { namespace constants { @@ -8,6 +10,17 @@ namespace constants { constexpr int seconds_per_day = 86400; constexpr int days_per_nonleap_year = 365; +// Universal fill value for variables +// TODO: When we switch to supporting C++17 we can use a simple `inline constexpr` rather than a struct +template +struct DefaultFillValue { + static const bool is_float = std::is_floating_point::value; + static const bool is_int = std::is_integral::value; + T value = is_int ? std::numeric_limits::max() / 2 : + is_float ? std::numeric_limits::max() / 1e5 : std::numeric_limits::max(); + +}; + } // namespace constants } // namespace scream diff --git a/components/eamxx/src/share/util/scream_utils.hpp b/components/eamxx/src/share/util/scream_utils.hpp index 86da734f591a..4dddebf75aa5 100644 --- a/components/eamxx/src/share/util/scream_utils.hpp +++ b/components/eamxx/src/share/util/scream_utils.hpp @@ -24,6 +24,15 @@ enum MemoryUnits { GiB }; +template +typename VT::HostMirror +cmvdc (const VT& v) +{ + auto vh = Kokkos::create_mirror_view(v); + Kokkos::deep_copy(vh,v); + return vh; +} + // Gets current memory (RAM) usage by current process. long long get_mem_usage (const MemoryUnits u); @@ -330,6 +339,14 @@ Int compare (const std::string& label, const Scalar* a, return nerr1 + nerr2; } +inline void +check_mpi_call (int err, const std::string& context) { + EKAT_REQUIRE_MSG (err==MPI_SUCCESS, + "Error! MPI operation encountered an error.\n" + " - err code: " + std::to_string(err) + "\n" + " - context: " + context + "\n"); +} + } // namespace scream #endif // SCREAM_UTILS_HPP diff --git a/components/eamxx/src/share/util/scream_vertical_interpolation.hpp b/components/eamxx/src/share/util/scream_vertical_interpolation.hpp index 79572cf6019c..68b9997c8595 100644 --- a/components/eamxx/src/share/util/scream_vertical_interpolation.hpp +++ b/components/eamxx/src/share/util/scream_vertical_interpolation.hpp @@ -104,6 +104,48 @@ void perform_vertical_interpolation( const int nlevs_tgt, const Real msk_val = masked_val); +template +void perform_vertical_interpolation( + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const int nlevs_src, + const int nlevs_tgt, + const Real msk_val = masked_val); + +template +void perform_vertical_interpolation( + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const view_Nd< Mask

,N>& mask, + const int nlevs_src, + const int nlevs_tgt, + const Real msk_val = masked_val); + +template +void perform_vertical_interpolation( + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const int nlevs_src, + const int nlevs_tgt, + const Real msk_val = masked_val); + +template +void perform_vertical_interpolation( + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const view_Nd< Mask

,N>& mask, + const int nlevs_src, + const int nlevs_tgt, + const Real msk_val = masked_val); + /* ---------------------------------------------------------------------- * Main interpolation routine that applies vertical interpolation to a * single vertical slice of data. @@ -124,7 +166,7 @@ void apply_interpolation_impl_1d( const LIV& vert_interp); /* ---------------------------------------------------------------------- - * Versions where x_tgt is a 2-D view + * Versions where x_src is a 2-D view and x_tgt is a 2-D view * ---------------------------------------------------------------------- */ template void perform_checks( @@ -160,7 +202,7 @@ void apply_interpolation( const view_3d< Mask

>& mask); /* ---------------------------------------------------------------------- - * Versions where x_tgt is a single 1-D vertical profile + * Versions where x_src is a 2-D view and x_tgt is a single 1-D vertical profile * ---------------------------------------------------------------------- */ template void perform_checks( @@ -195,6 +237,78 @@ void apply_interpolation( const view_3d< Pack>& output, const view_3d< Mask

>& mask); +/* ---------------------------------------------------------------------- + * Versions where x_src is a 1-D view and x_tgt is a 2-D view + * ---------------------------------------------------------------------- */ +template +void perform_checks( + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const int nlevs_src, + const int nlevs_tgt); + +template +void apply_interpolation( + const int num_levs_src, + const int num_levs_tgt, + const T mask_val, + const LIV& vert_interp, + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_2d>& input, + const view_2d< Pack>& output, + const view_2d< Mask

>& mask); + +template +void apply_interpolation( + const int num_levs_src, + const int num_levs_tgt, + const T mask_val, + const LIV& vert_interp, + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_3d>& input, + const view_3d< Pack>& output, + const view_3d< Mask

>& mask); + +/* ---------------------------------------------------------------------- + * Versions where x_src is a single 1-D view and x_tgt is a single 1-D vertical profile + * ---------------------------------------------------------------------- */ +template +void perform_checks( + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const int nlevs_src, + const int nlevs_tgt); + +template +void apply_interpolation( + const int num_levs_src, + const int num_levs_tgt, + const T mask_val, + const LIV& vert_interp, + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_2d>& input, + const view_2d< Pack>& output, + const view_2d< Mask

>& mask); + +template +void apply_interpolation( + const int num_levs_src, + const int num_levs_tgt, + const T mask_val, + const LIV& vert_interp, + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_3d>& input, + const view_3d< Pack>& output, + const view_3d< Mask

>& mask); + // Helper function to allocate memory for an Nd mask on the fly. template view_Nd,N> allocate_mask(const std::vector& extents); diff --git a/components/eamxx/src/share/util/scream_vertical_interpolation_impl.hpp b/components/eamxx/src/share/util/scream_vertical_interpolation_impl.hpp index 6dc279d81548..7b7007482b48 100644 --- a/components/eamxx/src/share/util/scream_vertical_interpolation_impl.hpp +++ b/components/eamxx/src/share/util/scream_vertical_interpolation_impl.hpp @@ -68,7 +68,7 @@ void apply_interpolation_impl_1d( } /* ---------------------------------------------------------------------- - * Versions where x_tgt is a 2-D view + * Versions where x_src is a 2-D view and x_tgt is a 2-D view * ---------------------------------------------------------------------- */ template void perform_checks( @@ -143,9 +143,6 @@ void perform_vertical_interpolation( const auto mask = allocate_mask(extents); LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - for (int ii=1; ii void perform_checks( @@ -348,6 +345,282 @@ void apply_interpolation( }); Kokkos::fence(); } + +/* ---------------------------------------------------------------------- + * Versions where x_src is a 1-D view and x_tgt is a 2-D view + * ---------------------------------------------------------------------- */ +template +void perform_checks( + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const int nlevs_src, + const int nlevs_tgt) +{ + auto rank = N; + EKAT_REQUIRE_MSG (rank>1 &&rank<=3,"Error::scream_vertical_interpolation, passed view of rank (" + std::to_string(rank) +"), only support ranks 2 or 3\n"); + + // The input data and x_src data should match in the appropriate size + EKAT_REQUIRE(x_src.extent_int(0) == input.extent_int(input.rank-1)); + // The output data and x_tgt data should match in the appropriate size + EKAT_REQUIRE(x_tgt.extent_int(0) == output.extent_int(0)); + EKAT_REQUIRE(x_tgt.extent_int(1) == output.extent_int(input.rank-1)); + + // The output data and the input data should match in all sizes except the last one + for (int ii=0;ii +void perform_vertical_interpolation( + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const view_Nd< Mask

,N>& mask, + const int nlevs_src, + const int nlevs_tgt, + const Real msk_val) +{ + int ndofs = 1; + for (int ii=0; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); + LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); + apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); +} + +template +void perform_vertical_interpolation( + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const int nlevs_src, + const int nlevs_tgt, + const Real msk_val) +{ + int ndofs = 1; + for (int ii=0; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); + + std::vector extents; + for (int ii=0;ii(extents); + + LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); + apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); +} + +template +void apply_interpolation( + const int num_levs_src, + const int num_levs_tgt, + const T mask_val, + const LIV& vert_interp, + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_2d>& input, + const view_2d< Pack>& output, + const view_2d< Mask

>& mask_out) +{ + const int d_0 = input.extent_int(0); + const int npacks = output.extent_int(output.rank-1); + const auto policy = ESU::get_default_team_policy(d_0, npacks); + Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, + KOKKOS_LAMBDA(MemberType const& team) { + + const int icol = team.league_rank(); + const auto x1 = x_src; + const auto xt = ekat::subview(x_tgt, icol); + const auto in = ekat::subview(input, icol); + const auto out = ekat::subview(output, icol); + const auto mask = ekat::subview(mask_out, icol); + + apply_interpolation_impl_1d(x1,xt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); + }); + Kokkos::fence(); +} + +template +void apply_interpolation( + const int num_levs_src, + const int num_levs_tgt, + const T mask_val, + const LIV& vert_interp, + const view_1d>& x_src, + const view_2d>& x_tgt, + const view_3d>& input, + const view_3d< Pack>& output, + const view_3d< Mask

>& mask_out) +{ + const int d_0 = input.extent_int(0); + const int num_vars = input.extent_int(1); + const int npacks = output.extent_int(output.rank-1); + const auto policy = ESU::get_default_team_policy(d_0*num_vars, npacks); + Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, + KOKKOS_LAMBDA(MemberType const& team) { + + const int icol = team.league_rank() / num_vars; + const int ivar = team.league_rank() % num_vars; + const auto x1 = x_src; + const auto xt = ekat::subview(x_tgt, icol); + const auto in = ekat::subview(input, icol, ivar); + const auto out = ekat::subview(output, icol, ivar); + const auto mask = ekat::subview(mask_out, icol, ivar); + + apply_interpolation_impl_1d(x1,xt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); + }); + Kokkos::fence(); +} + +/* ---------------------------------------------------------------------- + * Versions where x_src is a 1-D view and x_tgt is a single 1-D vertical profile + * ---------------------------------------------------------------------- */ +template +void perform_checks( + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const int nlevs_src, + const int nlevs_tgt) +{ + const int rank = input.rank; + EKAT_REQUIRE_MSG (rank>1 &&rank<=3,"Error::scream_vertical_interpolation, passed view of rank (" + std::to_string(rank) +"), only support ranks 2 or 3\n"); + + // The output and input data should match in rank + EKAT_REQUIRE(static_cast(output.rank)==rank); + // The output data and the input data should match in all sizes except the last one + for (int ii=0;ii +void perform_vertical_interpolation( + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const view_Nd< Mask

,N>& mask, + const int nlevs_src, + const int nlevs_tgt, + const Real msk_val) +{ + int ndofs = 1; + for (int ii=0; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); + LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); + apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); +} + +template +void perform_vertical_interpolation( + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_Nd,N>& input, + const view_Nd< Pack,N>& output, + const int nlevs_src, + const int nlevs_tgt, + const Real msk_val) +{ + int ndofs = 1; + for (int ii=1; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); + + std::vector extents; + for (int ii=0;ii(extents); + + LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); + apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); +} + +template +void apply_interpolation( + const int num_levs_src, + const int num_levs_tgt, + const T mask_val, + const LIV& vert_interp, + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_2d>& input, + const view_2d< Pack>& output, + const view_2d< Mask

>& mask_out) +{ + const int d_0 = input.extent_int(0); + const int npacks = output.extent_int(output.rank-1); + const auto policy = ESU::get_default_team_policy(d_0, npacks); + Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, + KOKKOS_LAMBDA(MemberType const& team) { + + const int icol = team.league_rank(); + const auto in = ekat::subview(input, icol); + const auto out = ekat::subview(output, icol); + const auto mask = ekat::subview(mask_out, icol); + + apply_interpolation_impl_1d(x_src,x_tgt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); + }); + Kokkos::fence(); +} + +template +void apply_interpolation( + const int num_levs_src, + const int num_levs_tgt, + const T mask_val, + const LIV& vert_interp, + const view_1d>& x_src, + const view_1d>& x_tgt, + const view_3d>& input, + const view_3d< Pack>& output, + const view_3d< Mask

>& mask_out) +{ + const int d_0 = input.extent_int(0); + const int num_vars = input.extent_int(1); + const int npacks = output.extent_int(output.rank-1); + const auto policy = ESU::get_default_team_policy(d_0*num_vars, npacks); + Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, + KOKKOS_LAMBDA(MemberType const& team) { + + const int icol = team.league_rank() / num_vars; + const int ivar = team.league_rank() % num_vars; + const auto in = ekat::subview(input, icol, ivar); + const auto out = ekat::subview(output, icol, ivar); + const auto mask = ekat::subview(mask_out, icol, ivar); + + apply_interpolation_impl_1d(x_src,x_tgt,in,out,mask,num_levs_src,num_levs_tgt,team.league_rank(),mask_val,team,vert_interp); + }); + Kokkos::fence(); +} } // namespace vinterp } // namespace scream diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index c1f959e10288..ab168e67e79f 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -1,9 +1,6 @@ include (ScreamUtils) -# Some tests for checking testing works -set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne4np4L72_20220822.nc") -set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") - +# Some tests for checking that certain testing infrastructures work add_subdirectory(generic) if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) @@ -12,6 +9,11 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) SetVarDependingOnTestSize(TEST_RANK_START ${SCREAM_TEST_MAX_RANKS} 1 1) set(TEST_RANK_END ${SCREAM_TEST_MAX_RANKS}) + # Initial condition files used in the tests + set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") + set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") + set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") + # Testing individual atm processes add_subdirectory(uncoupled) diff --git a/components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt index f4589ffb8e03..5127f42969a2 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt @@ -5,6 +5,11 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(model_restart) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_128levels) + add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_pg2) + add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_dp) + if (SCREAM_ENABLE_MAM) + add_subdirectory(homme_mam4xx_pg2) + endif() endif() endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt b/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt new file mode 100644 index 000000000000..bb24e93b28d5 --- /dev/null +++ b/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt @@ -0,0 +1,84 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_mam4xx_pg2) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 10) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS ${dynLibName} mam + LABELS dynamics physics mam4xx + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 4) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 6) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 2) +set (HOMME_TEST_TRANSPORT_ALG 12) +set (HOMME_TEST_CUBED_SPHERE_MAP 2) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 6) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/maps/map_ne4np4_to_ne2np4_mono.nc + scream/init/homme_mam4xx_ne4_init.nc + scream/init/${EAMxx_tests_IC_FILE_72lev} + cam/topo/${EAMxx_tests_TOPO_FILE} +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics mam4xx + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/input.yaml b/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/input.yaml new file mode 100644 index 000000000000..988e7b73f49a --- /dev/null +++ b/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/input.yaml @@ -0,0 +1,38 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/homme_mam4xx_ne4_init.nc + topography_filename: ${TOPO_DATA_DIR}/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc + cldfrac_tot : 0.0 + pbl_height : 1000.0 + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mam4_micro,mam4_optics] + schedule_type: Sequential + Type: Group + mam4_micro: + compute_tendencies : [q_aitken_so4, n_aitken, q_h2so4] + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml b/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml new file mode 100644 index 000000000000..23beda243aea --- /dev/null +++ b/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml @@ -0,0 +1,44 @@ +%YAML 1.1 +--- +filename_prefix: homme_mam4xx_pg2_output +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + - qv + - T_mid + # MAM4 microphysics + - q_h2so4 + - q_aitken_so4 + - n_aitken + # TODO: Diagnostics + # GLL output for homme states. These + # represent all current possible homme + # states available. + Dynamics: + Field Names: + - v_dyn + - vtheta_dp_dyn + - dp3d_dyn + - phi_int_dyn + - ps_dyn + - phis_dyn + - omega_dyn + - Qdp_dyn + IO Grid Name: Physics GLL + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt index 057dd894cd76..d4618934c7c4 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt @@ -1,16 +1,18 @@ include (ScreamUtils) +set (TEST_BASE_NAME homme_shoc_cld_p3_rrtmgp) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + # Get or create the dynamics lib # HOMME_TARGET NP PLEV QSIZE_D CreateDynamicsLib("theta-l_kokkos" 4 72 10) # Create the test -set (TEST_LABELS "dynamics;driver;shoc;cld;p3;rrtmgp;physics") -set (NEED_LIBS cld_fraction shoc p3 scream_rrtmgp ${dynLibName} scream_control scream_share physics_share diagnostics) -CreateUnitTest(homme_shoc_cld_p3_rrtmgp "homme_shoc_cld_p3_rrtmgp.cpp" "${NEED_LIBS}" - LABELS ${TEST_LABELS} +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction ${dynLibName} shoc p3 scream_rrtmgp + LABELS dynamics shoc cld p3 rrtmgp physics MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - PROPERTIES FIXTURES_SETUP homme_shoc_cld_p3_rrtmgp_generate_output_nc_files + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) # Set AD configurable options @@ -25,8 +27,8 @@ math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_M ## Copy (and configure) yaml files needed by tests configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/homme_shoc_cld_p3_rrtmgp_output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/homme_shoc_cld_p3_rrtmgp_output.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) # Set homme's test options, so that we can configure the namelist correctly # Discretization/algorithm settings @@ -38,11 +40,14 @@ set (HOMME_TEST_TIME_STEP 300) set (HOMME_THETA_FORM 1) set (HOMME_TTYPE 5) set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) # Hyperviscosity settings set (HOMME_TEST_HVSCALING 0) set (HOMME_TEST_HVS 1) set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) set (HOMME_TEST_NU 7e15) set (HOMME_TEST_NUDIV 1e15) @@ -62,27 +67,15 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) -GetInputFile(cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc) - -## Finally compare all MPI rank output files against the single rank output as a baseline, using CPRNC -## Only if running with 2+ ranks configurations -# This test requires CPRNC -if (TEST_RANK_END GREATER TEST_RANK_START) - include (BuildCprnc) - BuildCprnc() +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) - set (BASE_TEST_NAME "homme_shoc_cld_p3_rrtmgp") - math (EXPR CMP_RANK_START ${TEST_RANK_START}+1) - foreach (MPI_RANKS RANGE ${CMP_RANK_START} ${TEST_RANK_END}) +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) - set (SRC_FILE "${BASE_TEST_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${MPI_RANKS}.${RUN_T0}.nc") - set (TGT_FILE "${BASE_TEST_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_START}.${RUN_T0}.nc") - set (TEST_NAME "${BASE_TEST_NAME}_np${TEST_RANK_START}_vs_np${MPI_RANKS}_bfb") - add_test (NAME ${TEST_NAME} - COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(${TEST_NAME} PROPERTIES LABELS "${TEST_LABELS}" - RESOURCE_LOCK ${BASE_TEST_NAME} - FIXTURES_REQUIRED homme_shoc_cld_p3_rrtmgp_generate_output_nc_files) - endforeach() -endif() +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/homme_shoc_cld_p3_rrtmgp.cpp b/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/homme_shoc_cld_p3_rrtmgp.cpp deleted file mode 100644 index e76f2e888bf2..000000000000 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/homme_shoc_cld_p3_rrtmgp.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include "catch2/catch.hpp" - -// The AD -#include "control/atmosphere_driver.hpp" - -// Physics includes -#include "physics/p3/atmosphere_microphysics.hpp" -#include "physics/shoc/atmosphere_macrophysics.hpp" -#include "physics/cld_fraction/atmosphere_cld_fraction.hpp" -#include "physics/rrtmgp/atmosphere_radiation.hpp" -#include "diagnostics/register_diagnostics.hpp" - -// Dynamics includes -#include "dynamics/register_dynamics.hpp" -#include "dynamics/homme/interface/scream_homme_interface.hpp" - -// EKAT headers -#include "ekat/ekat_assert.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/ekat_assert.hpp" - -TEST_CASE("scream_homme_physics", "scream_homme_physics") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - ad_params.print(); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - // Register all atm procs and the grids manager in the respective factories - register_dynamics(); - auto& proc_factory = AtmosphereProcessFactory::instance(); - proc_factory.register_product("p3",&create_atmosphere_process); - proc_factory.register_product("SHOC",&create_atmosphere_process); - proc_factory.register_product("CldFraction",&create_atmosphere_process); - proc_factory.register_product("RRTMGP",&create_atmosphere_process); - register_diagnostics(); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - // NOTE: Kokkos is finalize in ekat_catch_main.cpp, and YAKL is finalized - // during RRTMGPRatiation::finalize_impl, after RRTMGP has deallocated - // all its arrays. - ad.initialize(atm_comm,ad_params,t0); - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - // Register all atm procs and the grids manager in the respective factories - register_dynamics(); - register_physics(); - register_diagnostics(); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - // NOTE: Kokkos is finalize in ekat_catch_main.cpp, and YAKL is finalized - // during RRTMGPRatiation::finalize_impl, after RRTMGP has deallocated - // all its arrays. - ad.initialize(atm_comm,ad_params,t0); - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - // Register all atm procs and the grids manager in the respective factories - register_dynamics(); - register_physics(); - register_diagnostics(); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - // NOTE: Kokkos is finalize in ekat_catch_main.cpp, and YAKL is finalized - // during RRTMGPRatiation::finalize_impl, after RRTMGP has deallocated - // all its arrays. - ad.initialize(atm_comm,ad_params,t0); - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i("time_step"); + const auto nsteps = ts.get("number_of_steps"); + const auto t0_str = ts.get("run_t0"); + const auto t0 = util::str_to_time_stamp(t0_str); + + // Register all atm procs and the grids manager in the respective factories + register_dynamics(); + register_physics(); + register_diagnostics(); + register_surface_coupling(); + + // Create the driver + AtmosphereDriver ad; + + // Init, run, and finalize + // NOTE: Kokkos is finalize in ekat_catch_main.cpp, and YAKL is finalized + // during RRTMGPRatiation::finalize_impl, after RRTMGP has deallocated + // all its arrays. + ad.set_comm(atm_comm); + ad.set_params(ad_params); + ad.init_scorpio (); + ad.init_time_stamps (t0, t0); + ad.create_atm_processes (); + ad.create_grids (); + ad.setup_intensive_observation_period (); + ad.create_fields (); + + // Setup surface coupler import to be NaNs for fields IOP should overwrite + const int ncols = ad.get_grids_manager()->get_grid("Physics")->get_num_local_dofs(); + static constexpr int num_imports = 4; + char import_names[num_imports][32]; + std::strcpy(import_names[0], "surf_radiative_T"); + std::strcpy(import_names[1], "surf_lw_flux_up"); + std::strcpy(import_names[2], "surf_sens_flux"); + std::strcpy(import_names[3], "surf_evap"); + KokkosTypes::view_2d import_data("import_data",ncols, num_imports); + Kokkos::deep_copy(import_data, std::nan("")); + KokkosTypes::view_1d import_cpl_indices("import_cpl_indices", num_imports); + std::iota(import_cpl_indices.data(), import_cpl_indices.data()+num_imports, 0); + KokkosTypes::view_1d import_vec_comps("import_vec_comps", num_imports); + Kokkos::deep_copy(import_vec_comps, -1); + KokkosTypes::view_1d import_constant_multiple("import_constant_multiple", num_imports); + Kokkos::deep_copy(import_constant_multiple, 1); + KokkosTypes::view_1d do_import_during_init("do_import_during_init", num_imports); + Kokkos::deep_copy(do_import_during_init, false); + do_import_during_init(2) = true; do_import_during_init(3) = true; + + ad.setup_surface_coupling_data_manager(SurfaceCouplingTransferType::Import, + 4, 4, ncols, import_data.data(), import_names[0], import_cpl_indices.data(), + import_vec_comps.data(), import_constant_multiple.data(), do_import_during_init.data()); + + ad.initialize_fields (); + ad.initialize_output_managers (); + ad.initialize_atm_procs (); + + if (atm_comm.am_i_root()) { + printf("Start time stepping loop... [ 0%%]\n"); + } + for (int i=0; i); - proc_factory.register_product("SHOC",&create_atmosphere_process); - proc_factory.register_product("CldFraction",&create_atmosphere_process); - proc_factory.register_product("RRTMGP",&create_atmosphere_process); - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto run_t0_str = ts.get("run_t0"); - const auto run_t0 = util::str_to_time_stamp(run_t0_str); - const auto case_t0_str = ts.get("case_t0"); - const auto case_t0 = util::str_to_time_stamp(case_t0_str); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - // NOTE: Kokkos is finalize in ekat_catch_main.cpp, and YAKL is finalized - // during RRTMGPRatiation::finalize_impl, after RRTMGP has deallocated - // all its arrays. - ad.initialize(atm_comm,ad_params,run_t0,case_t0); - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i - -// Boiler plate, needed for all runs -#include "control/atmosphere_driver.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" - -// Physics headers -#include "physics/p3/atmosphere_microphysics.hpp" -#include "physics/shoc/atmosphere_macrophysics.hpp" - -// EKAT headers -#include "ekat/ekat_pack.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/util/ekat_test_utils.hpp" - -#include - -namespace scream { - -TEST_CASE("shoc-p3", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - const auto& session = ekat::TestSession::get(); - std::string fname = session.params.at("ifile"); - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - // Need to register products in the factory *before* we create any atm process or grids manager. - auto& proc_factory = AtmosphereProcessFactory::instance(); - proc_factory.register_product("p3",&create_atmosphere_process); - proc_factory.register_product("SHOC",&create_atmosphere_process); - register_mesh_free_grids_manager(); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - ad.initialize(atm_comm,ad_params,t0); - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i - -// Boiler plate, needed for all runs -#include "control/atmosphere_driver.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" - -// Physics headers -#include "physics/p3/atmosphere_microphysics.hpp" -#include "physics/shoc/atmosphere_macrophysics.hpp" -#include "physics/cld_fraction/atmosphere_cld_fraction.hpp" -#include "physics/rrtmgp/atmosphere_radiation.hpp" - -// EKAT headers -#include "ekat/ekat_pack.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" - -namespace scream { - -TEST_CASE("shoc-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - // Need to register products in the factory *before* we create any atm process or grids manager. - auto& proc_factory = AtmosphereProcessFactory::instance(); - proc_factory.register_product("p3",&create_atmosphere_process); - proc_factory.register_product("SHOC",&create_atmosphere_process); - proc_factory.register_product("CldFraction",&create_atmosphere_process); - proc_factory.register_product("RRTMGP",&create_atmosphere_process); - register_mesh_free_grids_manager(); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - ad.initialize(atm_comm,ad_params,t0); - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i - -// Boiler plate, needed for all runs -#include "control/atmosphere_driver.hpp" - -#include "diagnostics/register_diagnostics.hpp" -#include "physics/register_physics.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" - -// EKAT headers -#include "ekat/ekat_parse_yaml_file.hpp" - -namespace scream { - -TEST_CASE("shoc-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - // Need to register products in the factory *before* we create any atm process or grids manager. - register_physics(); - register_diagnostics(); - register_mesh_free_grids_manager(); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - ad.initialize(atm_comm,ad_params,t0); - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i +#include "share/io/scream_output_manager.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +namespace { + +using namespace scream; + +void create_vert_remap() { + // Simple function to create a 1D remap column to test nudging w/ remapped data + ekat::Comm io_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. + MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); // MPI communicator group used for I/O. In our simple test we use MPI_COMM_WORLD, however a subset could be used. + scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler + + int nlevs = 5*SCREAM_PACK_SIZE+1; + std::vector dofs_levs(nlevs); + std::iota(dofs_levs.begin(),dofs_levs.end(),0); + std::vector p_tgt; + Real p_top=0, p_bot=102500; + Real dp = (p_bot - p_top) / (nlevs-1); + for (int ii=0; ii dofs(ntimes*ncols*nlevs); + std::iota(dofs.begin(),dofs.end(),0); + Real plev[nlevs]; + Real p_top=0, p_bot=102500; + Real dp = (p_bot - p_top) / (nlevs-1); + for (int ilev=0; ilev= 8.0e4) { + weights[itime][icol][ilev] = 1.; + } else { + weights[itime][icol][ilev] = 0.; + } + } + } + } + + scorpio::register_file(filename, scorpio::FileMode::Write); + scorpio::register_dimension(filename,"ncol", "ncol", ncols, false); + scorpio::register_dimension(filename,"lev", "lev", nlevs, false); + scorpio::register_dimension(filename,"time","time",ntimes, false); + scorpio::register_variable(filename,"nudging_weights","nudging_weights","none",{"lev", "ncol", "time"},"real","real","Real-lev"); + scorpio::set_dof(filename,"nudging_weights",dofs.size(),dofs.data()); + scorpio::eam_pio_enddef(filename); + scorpio::grid_write_data_array(filename,"nudging_weights",&weights[0][0][0],ntimes*ncols*nlevs); + scorpio::eam_pio_closefile(filename); + scorpio::eam_pio_finalize(); +} + +TEST_CASE("create_vert_remap_and_weights","create_vert_remap_and_weights") +{ + create_vert_remap(); + create_nudging_weights_ncfile(1, 218, 72, "nudging_weights.nc"); +} +} // end namespace diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml b/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml new file mode 100644 index 000000000000..2c9b58b678cc --- /dev/null +++ b/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml @@ -0,0 +1,60 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + schedule_type: Sequential + atm_procs_list: [shoc,p3,nudging] + p3: + max_total_ni: 720.0e3 + nudging: + nudging_filename: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.${RUN_T0}.nc] + nudging_fields: ["T_mid", "qv"] + nudging_timescale: 1000 + use_nudging_weights: true + nudging_weights_file: nudging_weights.nc + source_pressure_type: ${VERT_TYPE} + source_pressure_file: vertical_remap.nc ## Only used in the case of STATIC_1D_VERTICAL_PROFILE + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: [output_${POSTFIX}.yaml] +... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_source_data.yaml b/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_source_data.yaml new file mode 100644 index 000000000000..c0c3056312e2 --- /dev/null +++ b/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_source_data.yaml @@ -0,0 +1,52 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + schedule_type: Sequential + atm_procs_list: [shoc,p3] + p3: + max_total_ni: 720.0e3 + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 128 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_128lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: [output_${POSTFIX}.yaml,output_${POSTFIX}_remapped.yaml] +... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml b/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml new file mode 100644 index 000000000000..d4779131bcbb --- /dev/null +++ b/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml @@ -0,0 +1,39 @@ +%YAML 1.1 +--- +filename_prefix: shoc_p3_${POSTFIX}_nudged +Averaging Type: Instant +Max Snapshots Per File: 2 +Field Names: + - p_mid + - T_mid + - qv + - qc + - qr + - qi + - qm + - nc + - nr + - ni + - bm + - eff_radius_qc + - eff_radius_qi + - micro_liq_ice_exchange + - micro_vap_liq_exchange + - micro_vap_ice_exchange + - tke + - eddy_diff_mom + - sgs_buoy_flux + - inv_qc_relvar + - pbl_height + - surf_evap + - surf_mom_flux + - surf_sens_flux + - nccn + - ni_activated + - nc_nuceat_tend + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: false +... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml b/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml new file mode 100644 index 000000000000..4dde5dae2d5a --- /dev/null +++ b/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml @@ -0,0 +1,15 @@ +%YAML 1.1 +--- +filename_prefix: shoc_p3_${POSTFIX}_nudged_remapped +Averaging Type: Instant +Max Snapshots Per File: 2 +vertical_remap_file: vertical_remap.nc +Field Names: + - T_mid + - qv + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: false +... diff --git a/components/eamxx/tests/generic/CMakeLists.txt b/components/eamxx/tests/generic/CMakeLists.txt index 7a4dd5ecc8b8..73d4fbc9c17d 100644 --- a/components/eamxx/tests/generic/CMakeLists.txt +++ b/components/eamxx/tests/generic/CMakeLists.txt @@ -1,2 +1,4 @@ # Tests to ensure tests that should fail do indeed fail add_subdirectory(fail_check) +# Test for the BFB-checking hasher +add_subdirectory(bfbhash) diff --git a/components/eamxx/tests/generic/bfbhash/CMakeLists.txt b/components/eamxx/tests/generic/bfbhash/CMakeLists.txt new file mode 100644 index 000000000000..dc4139a67953 --- /dev/null +++ b/components/eamxx/tests/generic/bfbhash/CMakeLists.txt @@ -0,0 +1,2 @@ +include(ScreamUtils) +CreateUnitTest(bfbhash "bfbhash.cpp" LABELS "bfbhash") diff --git a/components/eamxx/tests/generic/bfbhash/bfbhash.cpp b/components/eamxx/tests/generic/bfbhash/bfbhash.cpp new file mode 100644 index 000000000000..5f049265f6ea --- /dev/null +++ b/components/eamxx/tests/generic/bfbhash/bfbhash.cpp @@ -0,0 +1,68 @@ +#include "share/util/scream_bfbhash.hpp" + +#include + +#include +#include +#include + +namespace scream { + +using namespace bfbhash; + +template +static void testeq () { + int cnt = 0; + for (const T perturb : {std::numeric_limits::epsilon(), T(-1e-3), + T(1e-2)*std::numeric_limits::epsilon()}) { + const T x = M_PI; + const T y = M_PI*(1 + perturb); + HashType a = 0, b = 0; + hash(x, a); + hash(y, b); + REQUIRE((a == b) == (x == y)); + if (a == b) ++cnt; + } + REQUIRE(cnt == 1); +} + +TEST_CASE("bfbhash") +{ + { // Repeated values should not be nullified. + HashType a = 0; + hash(M_PI, a); + hash(M_PI, a); + REQUIRE(a != 0); + } + + { // Negated values should not be nullified. + HashType a = 0; + hash( M_PI, a); + hash(-M_PI, a); + REQUIRE(a != 0); + } + + { // The hasher is sensitive to diffs that double accum is not. + double a = M_PI, b = 1e-20, c = a + b; + HashType x = 0, y = 0; + hash(a, x); + hash(a, y); hash(b, y); + REQUIRE(a == c); // double add doesn't see b + REQUIRE(x != y); // but the hasher does + } + + testeq(); + testeq(); + + { + ekat::Comm comm(MPI_COMM_WORLD); + HashType a = comm.rank(); + HashType b; + all_reduce_HashType(MPI_COMM_WORLD, &a, &b, 1); + HashType c = 0; + for (int i = 0, n = comm.size(); i < n; ++i) hash(HashType(i), c); + REQUIRE(b == c); + } +} + +} // namespace scream diff --git a/components/eamxx/tests/generic/fail_check/CMakeLists.txt b/components/eamxx/tests/generic/fail_check/CMakeLists.txt index f6a9fe4c583a..7c23f4fd52d5 100644 --- a/components/eamxx/tests/generic/fail_check/CMakeLists.txt +++ b/components/eamxx/tests/generic/fail_check/CMakeLists.txt @@ -4,23 +4,34 @@ include(ScreamUtils) # NOTE: we don't need any libs for this test, but scream's CreateUnitTest # has libs as a required arg. So use the raw EkatCreateUnitTest -# Ensure that a non-satisfied REQUIRE clause does, in fact, make the test fail -EkatCreateUnitTestFromExec (fail "fail.cpp" PROPERTIES WILL_FAIL TRUE LABELS "fail") +# This serves as a base case for the following fail checks, since it verifies that +# - a REQUIRE clause that fails makes the test fail +# - Our Create unit test logic does work for catching failures +CreateUnitTest (fail "fail.cpp" + PROPERTIES WILL_FAIL TRUE + LABELS "fail") if (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK) # Ensure that Kokkos OOB are caught - EkatCreateUnitTest (kokkos_fail "kokkos_fail.cpp" PROPERTIES WILL_FAIL TRUE LABELS "fail") + CreateUnitTest (kokkos_fail "kokkos_fail.cpp" + PROPERTIES WILL_FAIL TRUE + LABELS "fail") endif() if (EKAT_ENABLE_VALGRIND) # Ensure that valgrind errors are caught - EkatCreateUnitTest (valg_fail "valg_fail.cpp" PROPERTIES WILL_FAIL TRUE LABELS "fail") + EkatCreateUnitTest (valg_fail "valg_fail.cpp" + PROPERTIES WILL_FAIL TRUE + LABELS "fail") endif() # Ensure that FPE *do* throw when we expect them to -CreateUnitTestExec (scream_fpe_check "fpe_check.cpp" "scream_share") +CreateUnitTestExec (scream_fpe_check "fpe_check.cpp") if (SCREAM_FPE) - CreateUnitTestFromExec (scream_fpe_check scream_fpe_check PROPERTIES WILL_FAIL TRUE LABELS "check") + CreateUnitTestFromExec (scream_fpe_check scream_fpe_check + PROPERTIES WILL_FAIL TRUE + LABELS "check") else() - CreateUnitTestFromExec (scream_fpe_check scream_fpe_check LABELS "check") + CreateUnitTestFromExec (scream_fpe_check scream_fpe_check + LABELS "check") endif() diff --git a/components/eamxx/tests/generic/fail_check/fail.cpp b/components/eamxx/tests/generic/fail_check/fail.cpp index 092fcbe5432f..c6483a93e960 100644 --- a/components/eamxx/tests/generic/fail_check/fail.cpp +++ b/components/eamxx/tests/generic/fail_check/fail.cpp @@ -1,12 +1,6 @@ #include -#include - -namespace scream { - TEST_CASE("force_fail") { REQUIRE(false); // force this test to fail } - -} // empty namespace diff --git a/components/eamxx/tests/postrun/check_hashes_ers.py b/components/eamxx/tests/postrun/check_hashes_ers.py new file mode 100755 index 000000000000..be1da31ffff0 --- /dev/null +++ b/components/eamxx/tests/postrun/check_hashes_ers.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 + +import os, sys, re, glob, subprocess + +def readall(fn): + with open(fn,'r') as f: + txt = f.read() + return txt + +def greptxt(pattern, txt): + return re.findall('(?:' + pattern + ').*', txt, flags=re.MULTILINE) + +def grep(pattern, fn): + txt = readall(fn) + return greptxt(pattern, txt) + +def get_log_glob_from_atm_modelio(case_dir): + filename = case_dir + os.path.sep + 'CaseDocs' + os.path.sep + 'atm_modelio.nml' + ln = grep('diro = ', filename)[0] + run_dir = ln.split()[2].split('"')[1] + ln = grep('logfile = ', filename)[0] + atm_log_fn = ln.split()[2].split('"')[1] + id = atm_log_fn.split('.')[2] + return run_dir + os.path.sep + 'e3sm.log.' + id + '*' + +def get_hash_lines(fn): + rlns = subprocess.run(['zgrep', 'exxhash', fn], capture_output=True) + rlns = rlns.stdout.decode().split('\n') + lns = [] + if len(rlns) == 0: return lns + for rln in rlns: + pos = rln.find('exxhash') + lns.append(rln[pos:]) + return lns + +def parse_time(hash_ln): + return hash_ln.split()[1:3] + +def all_equal(t1, t2): + if len(t1) != len(t2): return False + for i in range(len(t1)): + if t1[i] != t2[i]: return False + return True + +def find_first_index_at_time(lns, time): + for i, ln in enumerate(lns): + t = parse_time(ln) + if all_equal(time, t): return i + return None + +def diff(l1, l2): + diffs = [] + for i in range(len(l1)): + if l1[i] != l2[i]: + diffs.append((l1[i], l2[i])) + return diffs + +def main(case_dir): + # Look for the two e3sm.log files. + glob_pat = get_log_glob_from_atm_modelio(case_dir) + e3sm_fns = glob.glob(glob_pat) + if len(e3sm_fns) == 0: + print('Could not find e3sm.log files with glob string {}'.format(glob_pat)) + return False + e3sm_fns.sort() + if len(e3sm_fns) == 1: + # This is the first run. Exit and wait for the second + # run. (POSTRUN_SCRIPT is called after each of the two runs.) + print('Exiting on first run.') + return True + print('Diffing base {} and restart {}'.format(e3sm_fns[0], e3sm_fns[1])) + + # Because of the prefixed 1: and 2: on some systems, we can't just use + # zdiff. + lns = [] + for f in e3sm_fns: + lns.append(get_hash_lines(f)) + time = parse_time(lns[1][0]) + time_idx = find_first_index_at_time(lns[0], time) + if time_idx is None: + print('Could not find a start time.') + return False + lns[0] = lns[0][time_idx:] + if len(lns[0]) != len(lns[1]): + print('Number of hash lines starting at restart time do not agree.') + return False + diffs = diff(lns[0], lns[1]) + + # Flushed prints to e3sm.log can sometimes conflict with other + # output. Permit up to 'thr' diffs so we don't fail due to badly printed + # lines. This isn't a big loss in checking because an ERS_Ln22 second run + # writes > 1000 hash lines, and a true loss of BFBness is nearly certain to + # propagate to a large number of subsequent hashes. + thr = 5 + if len(lns[0]) < 100: thr = 0 + + ok = True + if len(diffs) > thr: + print('DIFF') + print(diffs[-10:]) + ok = False + else: + print('OK') + + return ok + +case_dir = sys.argv[1] +sys.exit(0 if main(case_dir) else 1) diff --git a/components/eamxx/tests/uncoupled/CMakeLists.txt b/components/eamxx/tests/uncoupled/CMakeLists.txt index 1b9b9f571bd2..dd0fd493e29a 100644 --- a/components/eamxx/tests/uncoupled/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/CMakeLists.txt @@ -3,19 +3,22 @@ if (NOT SCREAM_BASELINES_ONLY) if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") add_subdirectory(homme) endif() - add_subdirectory(p3) add_subdirectory(shoc) add_subdirectory(cld_fraction) add_subdirectory(spa) add_subdirectory(surface_coupling) - if (RUN_ML_CORRECTION_TEST) + if (SCREAM_ENABLE_ML_CORRECTION ) add_subdirectory(ml_correction) endif() if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) + add_subdirectory(cosp) else() - message(STATUS "RRTMGP only supported for double precision builds; skipping") + message(STATUS "RRTMGP and COSP only supported for double precision builds; skipping") + endif() + if (SCREAM_ENABLE_MAM) + add_subdirectory(mam4) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/uncoupled/cld_fraction/CMakeLists.txt b/components/eamxx/tests/uncoupled/cld_fraction/CMakeLists.txt index 9cbb6c348337..442b1451b22b 100644 --- a/components/eamxx/tests/uncoupled/cld_fraction/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/cld_fraction/CMakeLists.txt @@ -1,9 +1,9 @@ include (ScreamUtils) -set (NEED_LIBS cld_fraction scream_control scream_share) - # Test atmosphere processes -CreateUnitTest(cld_fraction_standalone "cld_fraction_standalone.cpp" "${NEED_LIBS}" LABELS "cld_fraction;physics;driver") +CreateADUnitTest(cld_fraction_standalone + LIBS cld_fraction + LABELS cld_fraction physics) # Set AD configurable options set (NUM_STEPS 1) diff --git a/components/eamxx/tests/uncoupled/cld_fraction/cld_fraction_standalone.cpp b/components/eamxx/tests/uncoupled/cld_fraction/cld_fraction_standalone.cpp deleted file mode 100644 index a3ffdd563594..000000000000 --- a/components/eamxx/tests/uncoupled/cld_fraction/cld_fraction_standalone.cpp +++ /dev/null @@ -1,170 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" - -#include "physics/cld_fraction/atmosphere_cld_fraction.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" - -#include - -namespace scream { - -TEST_CASE("cld_fraction-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - // Cloud fraction specific parameters - const auto procs_params = ad_params.sublist("atmosphere_processes"); - const auto cld_params = procs_params.sublist("CldFraction"); - const auto ice_thresh = cld_params.get("ice_cloud_threshold"); - const auto ice_thresh_out = cld_params.get("ice_cloud_for_analysis_threshold"); - - EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Need to register products in the factory *before* we create any atm process or grids manager. - auto& proc_factory = AtmosphereProcessFactory::instance(); - auto& gm_factory = GridsManagerFactory::instance(); - proc_factory.register_product("CldFraction",&create_atmosphere_process); - gm_factory.register_product("Mesh Free",&create_mesh_free_grids_manager); - - // Create the driver - AtmosphereDriver ad; - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - - // Because this is a relatively simple test based on two variables, we initialize them here - // rather than use the netCDF input structure. - const auto& grid = ad.get_grids_manager()->get_grid("Point Grid"); - const auto& field_mgr = *ad.get_field_mgr(grid->name()); - - int num_cols = grid->get_num_local_dofs(); // Number of columns on this rank - int num_levs = grid->get_num_vertical_levels(); // Number of levels per column - - const auto& qi_field = field_mgr.get_field("qi"); - const auto& liq_cld_frac_field = field_mgr.get_field("cldfrac_liq"); - const auto& qi = qi_field.get_view(); - const auto& liq_cld_frac = liq_cld_frac_field.get_view(); - - // Set initial ice mass mixing ratio and liquid cloud fraction - const Real Pi = 3.14159265358979323846; - Real qi_amp; - Real cf_amp; - for (int icol=0;icol(); - const auto& tot_cld_frac = tot_cld_frac_field.get_view(); - - const auto& ice_cld_frac_field_4out = field_mgr.get_field("cldfrac_ice_for_analysis"); - const auto& tot_cld_frac_field_4out = field_mgr.get_field("cldfrac_tot_for_analysis"); - ice_cld_frac_field_4out.sync_to_host(); - tot_cld_frac_field_4out.sync_to_host(); - const auto& ice_cld_frac_4out = ice_cld_frac_field_4out.get_view(); - const auto& tot_cld_frac_4out = tot_cld_frac_field_4out.get_view(); - - for (int icol=0;icol1.0) or !(ice_cld_frac(icol,jlev)>1.0) or !(tot_cld_frac(icol,jlev)>1.0))); - REQUIRE((!(liq_cld_frac(icol,jlev)<0.0) or !(ice_cld_frac(icol,jlev)<0.0) or !(tot_cld_frac(icol,jlev)<0.0))); - // make sure that the cloud fraction calculation didn't accidentally change qi or liq_cld_fraction - Real phase = icol*Pi/2.0/(num_cols-1); - Real xval = jlev*Pi/2.0/(num_levs-1); - Real y_cmp = qi_amp*(1.0-std::sin(xval-phase))/2.0; - REQUIRE(qi(icol,jlev)==y_cmp); - y_cmp = cf_amp*(std::sin(xval+phase)+1.0)/2.0; - REQUIRE(liq_cld_frac(icol,jlev)==y_cmp); - // Test that the cloud fraction calculation appropriately calculated the ice cloud fraction - if (qi(icol,jlev)>ice_thresh) - { - REQUIRE(ice_cld_frac(icol,jlev)==1.0); - } else { - REQUIRE(ice_cld_frac(icol,jlev)==0.0); - } - if (qi(icol,jlev)>ice_thresh_out) - { - REQUIRE(ice_cld_frac_4out(icol,jlev)==1.0); - } else { - REQUIRE(ice_cld_frac_4out(icol,jlev)==0.0); - } - // Test that the total cloud fraction is correctly calculated - if (liq_cld_frac(icol,jlev) >= ice_cld_frac(icol,jlev)) - { - REQUIRE(tot_cld_frac(icol,jlev)==liq_cld_frac(icol,jlev)); - } else { - REQUIRE(tot_cld_frac(icol,jlev)==ice_cld_frac(icol,jlev)); - } - if (liq_cld_frac(icol,jlev) >= ice_cld_frac_4out(icol,jlev)) - { - REQUIRE(tot_cld_frac_4out(icol,jlev)==liq_cld_frac(icol,jlev)); - } else { - REQUIRE(tot_cld_frac_4out(icol,jlev)==ice_cld_frac_4out(icol,jlev)); - } - } - } - - // Finalize - ad.finalize(); - -} - -} // empty namespace diff --git a/components/eamxx/tests/uncoupled/cld_fraction/input.yaml b/components/eamxx/tests/uncoupled/cld_fraction/input.yaml index 5e1cefd367a5..7e7c154246bc 100644 --- a/components/eamxx/tests/uncoupled/cld_fraction/input.yaml +++ b/components/eamxx/tests/uncoupled/cld_fraction/input.yaml @@ -9,15 +9,19 @@ time_stepping: number_of_steps: ${NUM_STEPS} atmosphere_processes: - atm_procs_list: (CldFraction) + atm_procs_list: [CldFraction] CldFraction: ice_cloud_threshold: 1e-12 ice_cloud_for_analysis_threshold: 1e-5 grids_manager: Type: Mesh Free - number_of_global_columns: 3 - number_of_vertical_levels: 128 + grids_names: [Point Grid] + Point Grid: + aliases: [Physics] + type: point_grid + number_of_global_columns: 3 + number_of_vertical_levels: 128 initial_conditions: cldfrac_liq: 0.0 diff --git a/components/eamxx/tests/uncoupled/cosp/CMakeLists.txt b/components/eamxx/tests/uncoupled/cosp/CMakeLists.txt new file mode 100644 index 000000000000..769c6593aca8 --- /dev/null +++ b/components/eamxx/tests/uncoupled/cosp/CMakeLists.txt @@ -0,0 +1,22 @@ +include (ScreamUtils) + +# Create the test +CreateADUnitTest(cosp_standalone + LABELS cosp physics + LIBS eamxx_cosp + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} +) + +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) +GetInputFile(cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc) + +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) diff --git a/components/eamxx/tests/uncoupled/cosp/input.yaml b/components/eamxx/tests/uncoupled/cosp/input.yaml new file mode 100644 index 000000000000..3336b0694db5 --- /dev/null +++ b/components/eamxx/tests/uncoupled/cosp/input.yaml @@ -0,0 +1,40 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [cosp] + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics] + Physics: + aliases: [Point Grid] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + topography_filename: ${TOPO_DATA_DIR}/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc + dtau067: 1.0 + dtau105: 1.0 + cldfrac_tot_for_analysis: 0.5 + eff_radius_qc: 0.0 + eff_radius_qi: 0.0 + sunlit: 1.0 + surf_radiative_T: 288.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/uncoupled/cosp/output.yaml b/components/eamxx/tests/uncoupled/cosp/output.yaml new file mode 100644 index 000000000000..59cc18aeddf9 --- /dev/null +++ b/components/eamxx/tests/uncoupled/cosp/output.yaml @@ -0,0 +1,14 @@ +%YAML 1.1 +--- +filename_prefix: cosp_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - isccp_cldtot + +output_control: + Frequency: 1 + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/uncoupled/homme/CMakeLists.txt b/components/eamxx/tests/uncoupled/homme/CMakeLists.txt index c23efd6ac7cc..13dcdaea21b5 100644 --- a/components/eamxx/tests/uncoupled/homme/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/homme/CMakeLists.txt @@ -1,17 +1,18 @@ include (ScreamUtils) +set (TEST_BASE_NAME homme_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + # Get or create the dynamics lib # HOMME_TARGET NP PLEV QSIZE_D CreateDynamicsLib("theta-l_kokkos" 4 72 10) # Create the test -set (TEST_LABELS "dynamics;driver") -set (NEED_LIBS ${dynLibName} scream_control scream_share diagnostics) - -CreateUnitTest(homme_standalone "homme_standalone.cpp" "${NEED_LIBS}" - LABELS ${TEST_LABELS} +CreateADUnitTest(${TEST_BASE_NAME} + LABELS dynamics + LIBS ${dynLibName} MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - PROPERTIES FIXTURES_SETUP homme_generate_output_nc_files + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) # Set AD configurable options @@ -21,8 +22,8 @@ set (RUN_T0 2021-10-12-45000) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/homme_standalone_output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/homme_standalone_output.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) # Set homme's test options, so that we can configure the namelist correctly @@ -35,11 +36,14 @@ set (HOMME_TEST_TIME_STEP 300) set (HOMME_THETA_FORM 1) set (HOMME_TTYPE 5) set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) # Hyperviscosity settings set (HOMME_TEST_HVSCALING 0) set (HOMME_TEST_HVS 1) set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) set (HOMME_TEST_NU 7e15) set (HOMME_TEST_NUDIV 1e15) @@ -59,26 +63,17 @@ configure_file(${SCREAM_BASE_DIR}/src/dynamics/homme/tests/theta.nl # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) -GetInputFile(cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc) - -## Finally compare all MPI rank output files against the single rank output as a baseline, using CPRNC -## Only if running with 2+ ranks configurations -# This test requires CPRNC -if (TEST_RANK_END GREATER TEST_RANK_START) - include (BuildCprnc) - BuildCprnc() - set (BASE_TEST_NAME "homme") - math (EXPR CMP_RANK_START ${TEST_RANK_START}+1) - foreach (MPI_RANKS RANGE ${CMP_RANK_START} ${TEST_RANK_END}) - - set (SRC_FILE "${BASE_TEST_NAME}_standalone_output.INSTANT.nsteps_x${NUM_STEPS}.np${MPI_RANKS}.${RUN_T0}.nc") - set (TGT_FILE "${BASE_TEST_NAME}_standalone_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_START}.${RUN_T0}.nc") - set (TEST_NAME "${BASE_TEST_NAME}_np${TEST_RANK_START}_vs_np${MPI_RANKS}_bfb") - add_test (NAME ${TEST_NAME} - COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(${TEST_NAME} PROPERTIES LABELS "${TEST_LABELS}" - RESOURCE_LOCK ${BASE_TEST_NAME} - FIXTURES_REQUIRED homme_generate_output_nc_files) - endforeach() -endif() +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CreateRange(MpiRanks ${TEST_RANK_START} ${TEST_RANK_END}) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/uncoupled/homme/homme_standalone.cpp b/components/eamxx/tests/uncoupled/homme/homme_standalone.cpp deleted file mode 100644 index 7baf32a130de..000000000000 --- a/components/eamxx/tests/uncoupled/homme/homme_standalone.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include "catch2/catch.hpp" - -#include "control/atmosphere_driver.hpp" -#include "share/atm_process/atmosphere_process_group.hpp" -#include "diagnostics/register_diagnostics.hpp" -#include "dynamics/register_dynamics.hpp" -#include "dynamics/homme/atmosphere_dynamics.hpp" -#include "dynamics/homme/interface/scream_homme_interface.hpp" -#include "dynamics/homme/homme_dimensions.hpp" - -#include "ekat/ekat_assert.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/ekat_assert.hpp" - -// Hommexx includes -#include "Context.hpp" -#include "FunctorsBuffersManager.hpp" -#include "ElementsGeometry.hpp" -#include "TimeLevel.hpp" - -#include - -TEST_CASE("scream_homme_standalone", "scream_homme_standalone") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); - - // Need to register products in the factory *before* we create any AtmosphereProcessGroup, - register_dynamics(); - register_diagnostics(); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - ad.initialize(atm_comm,ad_params,t0); - - // Check that topography data from the FM matches Homme. - { - auto& geo = Homme::Context::singleton().get(); - auto phis = geo.m_phis; - - const auto& atm_process_group = ad.get_atm_processes(); - const auto& process = atm_process_group->get_process(0); - auto homme_process = std::dynamic_pointer_cast(process); - EKAT_REQUIRE_MSG (process, "Error! Cast to HommeDynamics failed.\n"); - - const auto phinh_i = homme_process->get_internal_field("phi_int_dyn","Dynamics").get_view(); - - int nelem = Homme::Context::singleton().get().num_elems(); - constexpr int NVL = HOMMEXX_NUM_PHYSICAL_LEV; - - Kokkos::parallel_for(Kokkos::RangePolicy<>(0,nelem*NP*NP), - KOKKOS_LAMBDA (const int idx) { - const int ie = idx/(NP*NP); - const int ip = (idx/NP)%NP; - const int jp = idx%NP; - EKAT_KERNEL_ASSERT(phinh_i(ie,ip,jp,NVL) == phis(ie,ip,jp)); - }); - } - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i= 3.11 or download it") @@ -11,12 +14,12 @@ endif() find_package(Python REQUIRED COMPONENTS Interpreter Development) -set(NEED_LIBS pybind11::pybind11 Python::Python ml_correction scream_control scream_share) -CreateUnitTest(ml_correction_standalone "ml_correction_standalone.cpp" "${NEED_LIBS}" LABELS "ml_correction;physics;driver") +CreateUnitTest(ml_correction_standalone "ml_correction_standalone.cpp" + LIBS pybind11::pybind11 Python::Python ml_correction scream_control scream_share + LABELS ml_correction physics driver) target_compile_definitions(ml_correction_standalone PRIVATE -DCUSTOM_SYS_PATH="${CMAKE_CURRENT_SOURCE_DIR}") target_include_directories(ml_correction_standalone SYSTEM PRIVATE ${PYTHON_INCLUDE_DIRS}) -message(STATUS "Python include dirs: ${PYTHON_INCLUDE_DIRS}") # Set AD configurable options set(NUM_STEPS 1) @@ -25,4 +28,4 @@ set (RUN_T0 2021-10-12-45000) # Configure yaml input file to run directory configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) diff --git a/components/eamxx/tests/uncoupled/ml_correction/input.yaml b/components/eamxx/tests/uncoupled/ml_correction/input.yaml index ac98d8f111eb..0bc979a1c238 100644 --- a/components/eamxx/tests/uncoupled/ml_correction/input.yaml +++ b/components/eamxx/tests/uncoupled/ml_correction/input.yaml @@ -9,12 +9,21 @@ time_stepping: number_of_steps: ${NUM_STEPS} atmosphere_processes: - atm_procs_list: (MLCorrection) - + atm_procs_list: [MLCorrection] + MLCorrection: + ML_model_path_tq: NONE + ML_model_path_uv: NONE + ML_model_path_sfc_fluxes: NONE + ML_output_fields: ["qv","T_mid"] + ML_correction_unit_test: True grids_manager: Type: Mesh Free - number_of_global_columns: 3 - number_of_vertical_levels: 128 + grids_names: [Physics] + Physics: + aliases: [Point Grid] + type: point_grid + number_of_global_columns: 3 + number_of_vertical_levels: 128 initial_conditions: qi: 0.0 diff --git a/components/eamxx/tests/uncoupled/ml_correction/ml_correction_standalone.cpp b/components/eamxx/tests/uncoupled/ml_correction/ml_correction_standalone.cpp index 8510024da968..8548831e4c68 100644 --- a/components/eamxx/tests/uncoupled/ml_correction/ml_correction_standalone.cpp +++ b/components/eamxx/tests/uncoupled/ml_correction/ml_correction_standalone.cpp @@ -1,22 +1,23 @@ +#include + +#include "control/atmosphere_driver.hpp" +#include "physics/register_physics.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" + +#include + #include #include #include -#include #include -#include "control/atmosphere_driver.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" -#include "physics/ml_correction/atmosphere_ml_correction.hpp" -#include "share/atm_process/atmosphere_process.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" - namespace scream { TEST_CASE("ml_correction-stand-alone", "") { using namespace scream; using namespace scream::control; namespace py = pybind11; + std::string fname = "input.yaml"; ekat::ParameterList ad_params("Atmosphere Driver"); parse_yaml_file(fname, ad_params); @@ -26,23 +27,23 @@ TEST_CASE("ml_correction-stand-alone", "") { const auto nsteps = ts.get("number_of_steps"); const auto t0_str = ts.get("run_t0"); const auto t0 = util::str_to_time_stamp(t0_str); + const auto ml = ad_params.sublist("atmosphere_processes").sublist("MLCorrection"); + const auto ML_model_tq_path = ml.get("ML_model_path_tq"); + const auto ML_model_uv_path = ml.get("ML_model_path_uv"); EKAT_ASSERT_MSG(dt > 0, "Error! Time step must be positive.\n"); ekat::Comm atm_comm(MPI_COMM_WORLD); - auto &proc_factory = AtmosphereProcessFactory::instance(); - auto &gm_factory = GridsManagerFactory::instance(); - proc_factory.register_product("MLCorrection", - &create_atmosphere_process); - gm_factory.register_product("Mesh Free", &create_mesh_free_grids_manager); + register_physics(); + register_mesh_free_grids_manager(); AtmosphereDriver ad; ad.initialize(atm_comm, ad_params, t0); - const auto &grid = ad.get_grids_manager()->get_grid("Point Grid"); - const auto &field_mgr = *ad.get_field_mgr(grid->name()); + const auto& grid = ad.get_grids_manager()->get_grid("Physics"); + const auto& field_mgr = *ad.get_field_mgr(grid->name()); int num_cols = grid->get_num_local_dofs(); int num_levs = grid->get_num_vertical_levels(); @@ -58,21 +59,26 @@ TEST_CASE("ml_correction-stand-alone", "") { } } qv_field.sync_to_dev(); - Real reference = qv(1, 10); - reference += 0.1; - Real reference2 = qv(1, 30); - reference2 += 0.1; - pybind11::scoped_interpreter guard{}; - pybind11::module sys = pybind11::module::import("sys"); + Real reference = 1e-4; + int fpe_mask = ekat::get_enabled_fpes(); + ekat::disable_all_fpes(); // required for importing numpy + if ( Py_IsInitialized() == 0 ) { + py::initialize_interpreter(); + } + py::module sys = pybind11::module::import("sys"); sys.attr("path").attr("insert")(1, CUSTOM_SYS_PATH); - auto py_correction = pybind11::module::import("test_correction"); - py::object ob1 = py_correction.attr("modify_view")( + auto py_correction = py::module::import("test_correction"); + py::object ML_model_tq = py_correction.attr("get_ML_model")(ML_model_tq_path); + py::object ML_model_uv = py_correction.attr("get_ML_model")(ML_model_uv_path); + py::object ob1 = py_correction.attr("modify_view")( py::array_t( num_cols * num_levs, qv.data(), py::str{}), - num_cols, num_levs); + num_cols, num_levs, ML_model_tq, ML_model_uv); py::gil_scoped_release no_gil; + ekat::enable_fpes(fpe_mask); REQUIRE(qv(1, 10) == reference); // This is the one that is modified - REQUIRE(qv(1, 30) != reference2); // This one should be unchanged + REQUIRE(qv(0, 10) != reference); // This one should be unchanged ad.finalize(); } + } // namespace scream diff --git a/components/eamxx/tests/uncoupled/ml_correction/test_correction.py b/components/eamxx/tests/uncoupled/ml_correction/test_correction.py index b3b7ae1f3ef1..054cf5539686 100644 --- a/components/eamxx/tests/uncoupled/ml_correction/test_correction.py +++ b/components/eamxx/tests/uncoupled/ml_correction/test_correction.py @@ -1,6 +1,46 @@ import numpy as np +import h5py +import xarray as xr +import fv3fit +from scream_run.steppers.machine_learning import ( + MultiModelAdapter, + predict, +) -def modify_view(data, Ncol, Nlev): +def get_ML_model(model_path): + if model_path == "NONE": + return None + config = MachineLearningConfig(models=[model_path]) + model = open_model(config) + return model + + +def sample_ML_prediction( + nz: int, input_data: np.ndarray, ML_model_tq: str, ML_model_uv: str +): + """ + This function is used to generate a sample ML prediction for the given input data. + We use a constant output predictor to generate the prediction. + """ + output_variables = ["qv"] + outputs = { + "qv": np.full(nz, 1e-4), + } + predictor = fv3fit.testing.ConstantOutputPredictor( + input_variables=["qv"], + output_variables=output_variables, + ) + predictor.set_outputs(**outputs) + model = MultiModelAdapter([predictor]) + if len(input_data.shape) < 2: + input_data = input_data[np.newaxis, :] + input_data = xr.Dataset({"qv": xr.DataArray(data=input_data, dims=["ncol", "z"])}) + output = predict(model, input_data, dt=1.0) + return output["qv"].values + + +def modify_view(data, Ncol, Nlev, model_tq, model_uv): data = np.reshape(data, (-1, Nlev)) - data[1, 10] += 0.1 + prediction = sample_ML_prediction(Nlev, data[1, :], model_tq, model_uv) + data[1, :] = prediction diff --git a/components/eamxx/tests/uncoupled/p3/CMakeLists.txt b/components/eamxx/tests/uncoupled/p3/CMakeLists.txt index f5791136f6b9..a155ebc4dc74 100644 --- a/components/eamxx/tests/uncoupled/p3/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/p3/CMakeLists.txt @@ -1,11 +1,14 @@ include (ScreamUtils) +set (TEST_BASE_NAME p3_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + # Create the test -SET (TEST_LABELS "p3;physics;driver") -set (NEED_LIBS p3 scream_control scream_share diagnostics) -CreateUnitTest(p3_standalone "p3_standalone.cpp" "${NEED_LIBS}" LABELS ${TEST_LABELS} +CreateADUnitTest(${TEST_BASE_NAME} + LABELS p3 physics + LIBS p3 MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - PROPERTIES FIXTURES_SETUP p3_generate_output_nc_files + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) # Set AD configurable options @@ -16,32 +19,22 @@ set (RUN_T0 2021-10-12-45000) ## Copy (and configure) yaml files needed by tests configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(p3_standalone_output.yaml p3_standalone_output.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) -## Finally compare all MPI rank output files against the single rank output as a baseline, using CPRNC -## Only if running with 2+ ranks configurations -# This test requires CPRNC -if (TEST_RANK_END GREATER TEST_RANK_START) - include (BuildCprnc) - BuildCprnc() - SET (BASE_TEST_NAME "p3") - math (EXPR CMP_RANK_START ${TEST_RANK_START}+1) - foreach (MPI_RANKS RANGE ${CMP_RANK_START} ${TEST_RANK_END}) - - set (SRC_FILE "${BASE_TEST_NAME}_standalone_output.INSTANT.nsteps_x1.np${MPI_RANKS}.${RUN_T0}.nc") - set (TGT_FILE "${BASE_TEST_NAME}_standalone_output.INSTANT.nsteps_x1.np${TEST_RANK_START}.${RUN_T0}.nc") - set (TEST_NAME "${BASE_TEST_NAME}_np${TEST_RANK_START}_vs_np${MPI_RANKS}_bfb") - add_test (NAME ${TEST_NAME} - COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(${TEST_NAME} PROPERTIES LABELS "${TEST_LABELS}" - RESOURCE_LOCK ${BASE_TEST_NAME} - FIXTURES_REQUIRED p3_generate_output_nc_files) - endforeach() -endif() +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS p3 physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) # Check tendency calculation foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) @@ -52,6 +45,6 @@ foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) COMMAND ${script} -f ${fname} -v qc T_mid -t p3_qc_tend p3_T_mid_tend WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties (${tname} PROPERTIES - FIXTURES_REQUIRED p3_generate_output_nc_files - LABELS "${TEST_LABELS}") + LABELS "p3;physics" + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) endforeach() diff --git a/components/eamxx/tests/uncoupled/p3/input.yaml b/components/eamxx/tests/uncoupled/p3/input.yaml index c73cf97e6916..1af25d382704 100644 --- a/components/eamxx/tests/uncoupled/p3/input.yaml +++ b/components/eamxx/tests/uncoupled/p3/input.yaml @@ -9,15 +9,19 @@ time_stepping: number_of_steps: ${NUM_STEPS} atmosphere_processes: - atm_procs_list: (p3) + atm_procs_list: [p3] p3: + max_total_ni: 740.0e3 compute_tendencies: [T_mid,qc] do_prescribed_ccn: false grids_manager: Type: Mesh Free - number_of_global_columns: 218 - number_of_vertical_levels: 72 # Will want to change to 128 when a valid unit test is available. + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 initial_conditions: # The name of the file containing the initial conditions for this test. @@ -27,5 +31,5 @@ initial_conditions: # The parameters for I/O control Scorpio: - output_yaml_files: ["p3_standalone_output.yaml"] + output_yaml_files: ["output.yaml"] ... diff --git a/components/eamxx/tests/uncoupled/p3/p3_standalone_output.yaml b/components/eamxx/tests/uncoupled/p3/output.yaml similarity index 91% rename from components/eamxx/tests/uncoupled/p3/p3_standalone_output.yaml rename to components/eamxx/tests/uncoupled/p3/output.yaml index 902f28c512e2..a83da3a3848c 100644 --- a/components/eamxx/tests/uncoupled/p3/p3_standalone_output.yaml +++ b/components/eamxx/tests/uncoupled/p3/output.yaml @@ -4,6 +4,7 @@ filename_prefix: p3_standalone_output Averaging Type: Instant Field Names: - T_mid + - T_mid_at_2m - T_prev_micro_step - qv - qc @@ -17,11 +18,13 @@ Field Names: - qv_prev_micro_step - eff_radius_qc - eff_radius_qi + - eff_radius_qr - precip_liq_surf_mass - precip_ice_surf_mass - micro_liq_ice_exchange - micro_vap_liq_exchange - micro_vap_ice_exchange + - rainfrac - p3_T_mid_tend - p3_qc_tend output_control: diff --git a/components/eamxx/tests/uncoupled/p3/p3_standalone.cpp b/components/eamxx/tests/uncoupled/p3/p3_standalone.cpp deleted file mode 100644 index ce1f8b6902f5..000000000000 --- a/components/eamxx/tests/uncoupled/p3/p3_standalone.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/p3/atmosphere_microphysics.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "ekat/ekat_parse_yaml_file.hpp" - -#include - -namespace scream { - -TEST_CASE("p3-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - // Need to register products in the factory *before* we create any atm process or grids manager. - auto& proc_factory = AtmosphereProcessFactory::instance(); - auto& gm_factory = GridsManagerFactory::instance(); - proc_factory.register_product("p3",&create_atmosphere_process); - gm_factory.register_product("Mesh Free",&create_mesh_free_grids_manager); - register_diagnostics(); - - // Create the driver - AtmosphereDriver ad; - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i0, "Error! Time step must be positive.\n"); // Need to register products in the factory *before* we create any atm process or grids manager. - auto& proc_factory = AtmosphereProcessFactory::instance(); - auto& gm_factory = GridsManagerFactory::instance(); - proc_factory.register_product("rrtmgp",&create_atmosphere_process); - gm_factory.register_product("Mesh Free",&create_mesh_free_grids_manager); + register_physics(); + register_mesh_free_grids_manager(); register_diagnostics(); // Create the driver @@ -98,17 +93,10 @@ TEST_CASE("rrtmgp-stand-alone", "") { } else if (i == 3) { REQUIRE(!views_are_equal(sw_flux_up_old, sw_flux_up)); } - } - // TODO: get the field repo from the driver, and go get (one of) - // the output(s) of SHOC, to check its numerical value (if possible) - - // Finalize + // Finalize ad.finalize(); - - // If we got here, we were able to run shoc - REQUIRE(true); } } // empty namespace diff --git a/components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone_unit.cpp index f70b4aaefb94..9f1ca00aaff8 100644 --- a/components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone_unit.cpp @@ -6,23 +6,23 @@ // Other rrtmgp specific code needed specifically for this test #include "physics/rrtmgp/rrtmgp_test_utils.hpp" #include "physics/rrtmgp/scream_rrtmgp_interface.hpp" -#include "physics/rrtmgp/atmosphere_radiation.hpp" -#include "mo_gas_concentrations.h" -#include "mo_garand_atmos_io.h" -#include "YAKL.h" - +#include "physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp" +#include "physics/register_physics.hpp" #include "physics/share/physics_constants.hpp" // scream share headers -#include "diagnostics/register_diagnostics.hpp" -#include "share/atm_process/atmosphere_process.hpp" #include "share/grid/mesh_free_grids_manager.hpp" #include "share/util/scream_common_physics_functions.hpp" // EKAT headers -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" +#include +#include +#include + +// RRTMGP and YAKL +#include +#include +#include // System headers #include @@ -73,11 +73,8 @@ namespace scream { ekat::Comm atm_comm (MPI_COMM_WORLD); // Need to register products in the factory *before* we create any atm process or grids manager. - auto& proc_factory = AtmosphereProcessFactory::instance(); - auto& gm_factory = GridsManagerFactory::instance(); - proc_factory.register_product("RRTMGP",&create_atmosphere_process); - gm_factory.register_product("Mesh Free",&create_mesh_free_grids_manager); - register_diagnostics(); + register_physics (); + register_mesh_free_grids_manager(); // Create the driver AtmosphereDriver ad; @@ -134,7 +131,6 @@ namespace scream { auto ncol_all = grid->get_num_global_dofs(); auto p_lay_all = real2d("p_lay", ncol_all, nlay); auto t_lay_all = real2d("t_lay", ncol_all, nlay); - auto p_del_all = real2d("p_del", ncol_all, nlay); auto p_lev_all = real2d("p_lev", ncol_all, nlay+1); auto t_lev_all = real2d("t_lev", ncol_all, nlay+1); auto sfc_alb_dir_vis_all = real1d("sfc_alb_dir_vis", ncol_all); @@ -178,7 +174,6 @@ namespace scream { auto icol_all = ncol * irank + icol; p_lay(icol,ilay) = p_lay_all(icol_all,ilay); t_lay(icol,ilay) = t_lay_all(icol_all,ilay); - p_del(icol,ilay) = p_del_all(icol_all,ilay); lwp(icol,ilay) = lwp_all(icol_all,ilay); iwp(icol,ilay) = iwp_all(icol_all,ilay); rel(icol,ilay) = rel_all(icol_all,ilay); @@ -193,7 +188,6 @@ namespace scream { // Free temporary variables p_lay_all.deallocate(); t_lay_all.deallocate(); - p_del_all.deallocate(); p_lev_all.deallocate(); t_lev_all.deallocate(); sfc_alb_dir_vis_all.deallocate(); @@ -216,6 +210,7 @@ namespace scream { // NOTE: test problem provides lwp/iwp in g/m2, not kg/m2! Factor of 1e3 here! using physconst = scream::physics::Constants; auto qc = real2d("qc", ncol, nlay); + auto nc = real2d("nc", ncol, nlay); auto qi = real2d("qi", ncol, nlay); yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay, ncol), YAKL_LAMBDA(int ilay, int icol) { qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * physconst::gravit / p_del(icol,ilay); @@ -247,6 +242,7 @@ namespace scream { auto d_sfc_alb_dif_nir = field_mgr.get_field("sfc_alb_dif_nir").get_view(); auto d_surf_lw_flux_up = field_mgr.get_field("surf_lw_flux_up").get_view(); auto d_qc = field_mgr.get_field("qc").get_view(); + auto d_nc = field_mgr.get_field("nc").get_view(); auto d_qi = field_mgr.get_field("qi").get_view(); auto d_rel = field_mgr.get_field("eff_radius_qc").get_view(); auto d_rei = field_mgr.get_field("eff_radius_qi").get_view(); @@ -284,6 +280,7 @@ namespace scream { d_tmid(i,k) = t_lay(i+1,k+1); d_pdel(i,k) = p_del(i+1,k+1); d_qc(i,k) = qc(i+1,k+1); + d_nc(i,k) = nc(i+1,k+1); d_qi(i,k) = qi(i+1,k+1); d_rel(i,k) = rel(i+1,k+1); d_rei(i,k) = rei(i+1,k+1); @@ -394,6 +391,7 @@ namespace scream { rei.deallocate(); cld.deallocate(); qc.deallocate(); + nc.deallocate(); qi.deallocate(); mu0.deallocate(); gas_vmr.deallocate(); @@ -417,8 +415,5 @@ namespace scream { // RRTMGPRadiation::finalize_impl after RRTMGP has had the // opportunity to deallocate all it's arrays. ad.finalize(); - - // If we got this far, we were able to run the code through the AD - REQUIRE(true); } } diff --git a/components/eamxx/tests/uncoupled/shoc/CMakeLists.txt b/components/eamxx/tests/uncoupled/shoc/CMakeLists.txt index 6e2adcd13fd2..bebc54448ca0 100644 --- a/components/eamxx/tests/uncoupled/shoc/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/shoc/CMakeLists.txt @@ -1,11 +1,14 @@ include (ScreamUtils) +set (TEST_BASE_NAME shoc_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + # Create the test -SET (TEST_LABELS "shoc;physics;driver") -set (NEED_LIBS shoc scream_control scream_share diagnostics) -CreateUnitTest(shoc_standalone "shoc_standalone.cpp" "${NEED_LIBS}" LABELS ${TEST_LABELS} +CreateADUnitTest(${TEST_BASE_NAME} + LABELS shoc physics + LIBS shoc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - PROPERTIES FIXTURES_SETUP shoc_generate_output_nc_files + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) # Set AD configurable options @@ -19,34 +22,23 @@ math (EXPR NUM_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_D # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) -GetInputFile(cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(shoc_standalone_output.yaml shoc_standalone_output.yaml) - -## Finally compare all MPI rank output files against the single rank output as a baseline, using CPRNC -## Only if running with 2+ ranks configurations -# This test requires CPRNC -if (TEST_RANK_END GREATER TEST_RANK_START) - include (BuildCprnc) - BuildCprnc() - SET (BASE_TEST_NAME "shoc") - math (EXPR CMP_RANK_START ${TEST_RANK_START}+1) - foreach (MPI_RANKS RANGE ${CMP_RANK_START} ${TEST_RANK_END}) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) - set (SRC_FILE "${BASE_TEST_NAME}_standalone_output.INSTANT.nsteps_x1.np${MPI_RANKS}.${RUN_T0}.nc") - set (TGT_FILE "${BASE_TEST_NAME}_standalone_output.INSTANT.nsteps_x1.np${TEST_RANK_START}.${RUN_T0}.nc") +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) - set (TEST_NAME "${BASE_TEST_NAME}_np${TEST_RANK_START}_vs_np${MPI_RANKS}_bfb") - add_test (NAME ${TEST_NAME} - COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(${TEST_NAME} PROPERTIES LABELS "${TEST_LABELS}" - RESOURCE_LOCK ${BASE_TEST_NAME} - FIXTURES_REQUIRED shoc_generate_output_nc_files) - endforeach() -endif() +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS spa physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) # Check tendency calculation foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) @@ -59,8 +51,8 @@ foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) -t shoc_T_mid_tend shoc_qv_tend shoc_tke_tend shoc_horiz_winds_tend shoc_sgs_buoy_flux_tend shoc_eddy_diff_mom_tend shoc_qc_tend shoc_cldfrac_liq_tend WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties (${tname} PROPERTIES - FIXTURES_REQUIRED shoc_generate_output_nc_files - LABELS "${TEST_LABELS}") + LABELS "shoc;physics" + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) endforeach() # Check that the content of sliced vars U/V and corresponding surf_mom_flux @@ -73,14 +65,15 @@ foreach (RANK RANGE ${TEST_RANK_START} ${TEST_RANK_END}) -s ${nc_file} -c "horiz_winds(:,:,1,:)=U(:,:,:)" "horiz_winds(:,:,2,:)=V(:,:,:)" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties (check_U_V_slices_np${RANK} PROPERTIES - FIXTURES_REQUIRED shoc_generate_output_nc_files) + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${RANK}_omp1) add_test (NAME check_surf_mom_flux_slices_np${RANK} COMMAND ${SCREAM_BASE_DIR}/scripts/compare-nc-files -s ${nc_file} -c "surf_mom_flux(:,:,1)=surf_mom_flux_U(:,:)" "surf_mom_flux(:,:,2)=surf_mom_flux_V(:,:)" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties (check_surf_mom_flux_slices_np${RANK} PROPERTIES - FIXTURES_REQUIRED shoc_generate_output_nc_files) + LABELS "shoc;physics" + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${RANK}_omp1) endforeach() ################################ @@ -97,7 +90,7 @@ add_test (NAME check_U_V_slices_fail_diff WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties (check_U_V_slices_fail_diff PROPERTIES WILL_FAIL TRUE - FIXTURES_REQUIRED shoc_generate_output_nc_files) + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${TEST_RANK_START}_omp1) # Wrong layout add_test (NAME check_U_V_slices_fail_layout @@ -106,7 +99,7 @@ add_test (NAME check_U_V_slices_fail_layout WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties (check_U_V_slices_fail_layout PROPERTIES WILL_FAIL TRUE - FIXTURES_REQUIRED shoc_generate_output_nc_files) + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${TEST_RANK_START}_omp1) # Missing variable(s) add_test (NAME check_U_V_slices_fail_missing @@ -115,4 +108,4 @@ add_test (NAME check_U_V_slices_fail_missing WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties (check_U_V_slices_fail_missing PROPERTIES WILL_FAIL TRUE - FIXTURES_REQUIRED shoc_generate_output_nc_files) + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${TEST_RANK_START}_omp1) diff --git a/components/eamxx/tests/uncoupled/shoc/input.yaml b/components/eamxx/tests/uncoupled/shoc/input.yaml index b585b0e62510..befb43d98a09 100644 --- a/components/eamxx/tests/uncoupled/shoc/input.yaml +++ b/components/eamxx/tests/uncoupled/shoc/input.yaml @@ -9,25 +9,41 @@ time_stepping: number_of_steps: ${NUM_STEPS} atmosphere_processes: - atm_procs_list: (shoc) + atm_procs_list: [shoc] shoc: number_of_subcycles: ${NUM_SUBCYCLES} compute_tendencies: [all] + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 grids_manager: Type: Mesh Free - number_of_global_columns: 218 - number_of_vertical_levels: 72 # Will want to change to 128 when a valid unit test is available. geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} - topography_filename: ${TOPO_DATA_DIR}/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} surf_sens_flux: 0.0 surf_evap: 0.0 # The parameters for I/O control Scorpio: - output_yaml_files: ["shoc_standalone_output.yaml"] + output_yaml_files: ["output.yaml"] ... diff --git a/components/eamxx/tests/uncoupled/shoc/shoc_standalone_output.yaml b/components/eamxx/tests/uncoupled/shoc/output.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/shoc/shoc_standalone_output.yaml rename to components/eamxx/tests/uncoupled/shoc/output.yaml diff --git a/components/eamxx/tests/uncoupled/shoc/shoc_standalone.cpp b/components/eamxx/tests/uncoupled/shoc/shoc_standalone.cpp deleted file mode 100644 index 6bb6affb9dff..000000000000 --- a/components/eamxx/tests/uncoupled/shoc/shoc_standalone.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/shoc/atmosphere_macrophysics.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "ekat/ekat_parse_yaml_file.hpp" - -#include - -namespace scream { - -TEST_CASE("shoc-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params) ; - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); - - // Need to register products in the factory *before* we create any atm process or grids manager. - auto& proc_factory = AtmosphereProcessFactory::instance(); - auto& gm_factory = GridsManagerFactory::instance(); - proc_factory.register_product("shoc",&create_atmosphere_process); - gm_factory.register_product("Mesh Free",&create_mesh_free_grids_manager); - register_diagnostics(); - - // Create the driver - AtmosphereDriver ad; - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/spa/atmosphere_prescribed_aerosol.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" - -#include - -namespace scream { - -TEST_CASE("spa-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); - - // Need to register products in the factory *before* we create any atm process or grids manager. - auto& proc_factory = AtmosphereProcessFactory::instance(); - auto& gm_factory = GridsManagerFactory::instance(); - proc_factory.register_product("SPA",&create_atmosphere_process); - gm_factory.register_product("Mesh Free",&create_mesh_free_grids_manager); - register_diagnostics(); - - // Create the grids manager - auto& gm_params = ad_params.sublist("grids_manager"); - const std::string& gm_type = gm_params.get("Type"); - auto gm = GridsManagerFactory::instance().create(gm_type,atm_comm,gm_params); - - // Create the driver - AtmosphereDriver ad; - - // Init, run, and finalize - ad.initialize(atm_comm,ad_params,t0); - if (atm_comm.am_i_root()) { - printf("Start time stepping loop... [ 0%%]\n"); - } - for (int i=0; i #include #include namespace scream { +using vos_type = std::vector; +using vor_type = std::vector; +constexpr Real test_tol = std::numeric_limits::epsilon()*1e4; +constexpr Real FillValue = -99999.0; + +// Test function for prescribed values +Real test_func(const int col, const int t) { + return (col + 1 + t); +} + +// Wrapper for output manager that also extracts the list of files +class OutputManager4Test : public scream::OutputManager +{ +public: + OutputManager4Test() = default; + + void runme(const util::TimeStamp& ts) { + run(ts); + update_file_list(); + } + + std::vector get_list_of_files() { return m_list_of_files; } +private: + void update_file_list() { + if (std::find(m_list_of_files.begin(),m_list_of_files.end(), m_output_file_specs.filename) == m_list_of_files.end()) { + m_list_of_files.push_back(m_output_file_specs.filename); + } + } + std::vector m_list_of_files; +}; + +std::vector create_from_file_test_data(const ekat::Comm& comm, const util::TimeStamp& t0, const int ncols ) +{ + // Create a grids manager on the fly + ekat::ParameterList gm_params; + gm_params.set("grids_names",vos_type{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_type{"Physics"}); + pl.set("number_of_global_columns",ncols); + pl.set("number_of_vertical_levels",1); // We don't care about levels for a surface only file + auto gm = create_mesh_free_grids_manager(comm,gm_params); + gm->build_grids(); + // Create a fields manager on the fly with the appropriate fields and grid. + using namespace ekat::units; + using namespace ShortFieldTagsNames; + const auto grid = gm->get_grid("Physics"); + const int nlcols = grid->get_num_local_dofs(); + const auto dofs_gids = grid->get_dofs_gids().get_view(); + std::vector fnames = {"lwdn"}; + FieldLayout layout({COL},{nlcols}); + auto fm = std::make_shared(grid); + fm->registration_begins(); + fm->registration_ends(); + auto nondim = Units::nondimensional(); + for (auto name : fnames) { + FieldIdentifier fid(name,layout,nondim,grid->name()); + Field f(fid); + f.allocate_view(); + // Initialize data + auto f_view_h = f.get_view(); + for (int ii=0; iiadd_field(f); + } + + // Create output manager to handle the data + scorpio::eam_init_pio_subsystem(comm); + ekat::ParameterList om_pl; + om_pl.set("MPI Ranks in Filename",true); + om_pl.set("filename_prefix",std::string("surface_coupling_forcing")); + om_pl.set("Field Names",fnames); + om_pl.set("Averaging Type", std::string("INSTANT")); + om_pl.set("Max Snapshots Per File",2); + om_pl.set("fill_value",FillValue); + auto& ctrl_pl = om_pl.sublist("output_control"); + ctrl_pl.set("frequency_units",std::string("nsteps")); + ctrl_pl.set("Frequency",1); + ctrl_pl.set("MPI Ranks in Filename",true); + ctrl_pl.set("save_grid_data",false); + OutputManager4Test om; + om.setup(comm,om_pl,fm,gm,t0,false); + // Create output data: + // T=3600, well above the max timestep for the test. + auto tw = t0; + const int dt = 3600; + for (auto name : fnames) { + auto field = fm->get_field(name); + // Note we only care about surface values so we only need to generate data over nlcols. + auto f_view_h = field.get_view(); + for (int ii=0; ii::view_1d& import_cpl_indices_view, @@ -43,7 +154,9 @@ void setup_import_and_export_data( { std::vector import_order(num_cpl_imports); for (int f=0; f export_order(num_cpl_exports); for (int f=0; f("run_t0"); const auto t0 = util::str_to_time_stamp(t0_str); + const auto gmp = ad_params.sublist("grids_manager"); + const auto grid_name = gmp.get("grids_names"); + const auto gdp = gmp.sublist(grid_name[0]); + const auto ncol_in = gdp.get("number_of_global_columns"); // Set two export fields to be randomly set to a constant // This requires us to add a sublist to the parsed AD params yaml list. - using vos_type = std::vector; - using vor_type = std::vector; std::uniform_real_distribution pdf_real_constant_data(0.0,1.0); + + auto& ap_params = ad_params.sublist("atmosphere_processes"); + auto& sc_exp_params = ap_params.sublist("SurfaceCouplingExporter"); + // Set up forcing to a constant value const Real Faxa_swndf_const = pdf_real_constant_data(engine); const Real Faxa_swvdf_const = pdf_real_constant_data(engine); const vos_type exp_const_fields = {"Faxa_swndf","Faxa_swvdf"}; - const vor_type exp_const_values = {Faxa_swndf_const,Faxa_swvdf_const}; - auto& ap_params = ad_params.sublist("atmosphere_processes"); - auto& sc_exp_params = ap_params.sublist("SurfaceCouplingExporter"); + vor_type exp_const_values = {Faxa_swndf_const,Faxa_swvdf_const}; + atm_comm.broadcast(exp_const_values.data(),2,0); auto& exp_const_params = sc_exp_params.sublist("prescribed_constants"); exp_const_params.set("fields",exp_const_fields); exp_const_params.set("values",exp_const_values); + // Set up forcing to data interpolated from file + const auto exp_file_files = create_from_file_test_data(atm_comm, t0, ncol_in); + const vos_type exp_file_fields = {"Faxa_lwdn"}; + // Test the use of an alternative name as stored in the data file(s). + const vos_type exp_file_fields_alt_name = {"Faxa_lwdn:lwdn"}; + auto& exp_file_params = sc_exp_params.sublist("prescribed_from_file"); + exp_file_params.set("fields",exp_file_fields); + exp_file_params.set("fields_alt_name",exp_file_fields_alt_name); + exp_file_params.set("files",exp_file_files); // Need to register products in the factory *before* we create any atm process or grids manager. auto& proc_factory = AtmosphereProcessFactory::instance(); - auto& gm_factory = GridsManagerFactory::instance(); proc_factory.register_product("SurfaceCouplingImporter",&create_atmosphere_process); proc_factory.register_product("SurfaceCouplingExporter",&create_atmosphere_process); - gm_factory.register_product("Mesh Free",&create_mesh_free_grids_manager); + register_mesh_free_grids_manager(); register_diagnostics(); // Create the AD @@ -379,16 +512,15 @@ TEST_CASE("surface-coupling", "") { ad.create_grids (); ad.create_fields (); - const int ncols = ad.get_grids_manager()->get_grid("Physics")->get_num_local_dofs(); - - // Create test data for SurfaceCouplingDataManager + const int ncols = ad.get_grids_manager()->get_grid("Physics")->get_num_local_dofs(); // Create engine and pdfs for random test data std::uniform_int_distribution pdf_int_additional_fields(0,10); std::uniform_int_distribution pdf_int_dt(1,1800); std::uniform_real_distribution pdf_real_import_data(0.0,1.0); // Set up random value for dt - const int dt = pdf_int_dt(engine); + int dt = pdf_int_dt(engine); + atm_comm.broadcast(&dt,1,0); // Setup views to test import/export. For this test we consider a random number of non-imported/exported // cpl fields (in addition to the required scream imports/exports), then assign a random, non-repeating // cpl index for each field in [0, num_cpl_fields). @@ -396,6 +528,7 @@ TEST_CASE("surface-coupling", "") { const int num_scream_exports = 17; KokkosTypes::view_1d additional_import_exports("additional_import_exports", 2); ekat::genRandArray(additional_import_exports, engine, pdf_int_additional_fields); + atm_comm.broadcast(additional_import_exports.data(),2,0); const int num_additional_imports = additional_import_exports(0); const int num_additional_exports = additional_import_exports(1); const int num_cpl_imports = num_scream_imports + num_additional_imports; @@ -413,6 +546,7 @@ TEST_CASE("surface-coupling", "") { num_scream_imports); // Set import data to random (0,1) values ekat::genRandArray(import_data_view, engine, pdf_real_import_data); + atm_comm.broadcast(import_data_view.data(), num_cpl_imports, 0); // Set import names char import_names[num_scream_imports][32]; std::strcpy(import_names[0], "sfc_alb_dir_vis"); @@ -467,7 +601,8 @@ TEST_CASE("surface-coupling", "") { // Setup the import/export data. This is meant to replicate the structures coming // from mct_coupling/scream_cpl_indices.F90 - setup_import_and_export_data(num_cpl_imports, num_scream_imports, + setup_import_and_export_data(engine, atm_comm, + num_cpl_imports, num_scream_imports, import_cpl_indices_view, import_vec_comps_view, import_constant_multiple_view, do_import_during_init_view, num_cpl_exports, num_scream_exports, @@ -508,10 +643,6 @@ TEST_CASE("surface-coupling", "") { // Finalize the AD ad.finalize(); - - // If we got here, we were able to run surface_coupling - REQUIRE(true); } - } // empty namespace diff --git a/components/eamxx/tests/uncoupled/zm/input.yaml b/components/eamxx/tests/uncoupled/zm/input.yaml index 045d5bedd09a..dd0344e5bcb2 100644 --- a/components/eamxx/tests/uncoupled/zm/input.yaml +++ b/components/eamxx/tests/uncoupled/zm/input.yaml @@ -9,10 +9,13 @@ time_stepping: number_of_steps: ${NUM_STEPS} atmosphere_processes: - atm_procs_list: (zm) + atm_procs_list: [zm] grids_manager: Type: Mesh Free - number_of_global_columns: 32 - number_of_vertical_levels: 128 + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 32 + number_of_vertical_levels: 128 ... diff --git a/components/eamxx/tpls/CMakeLists.txt b/components/eamxx/tpls/CMakeLists.txt index 6ee548ec9483..d89f48d4cacd 100644 --- a/components/eamxx/tpls/CMakeLists.txt +++ b/components/eamxx/tpls/CMakeLists.txt @@ -4,6 +4,7 @@ # - wrap pre-built library in a CMake target (CIME build) # First pioc/piof, since we link against it in csm_share (at least in CIME build) + include (${SCREAM_BASE_DIR}/cmake/tpls/Scorpio.cmake) CreateScorpioTargets() @@ -11,10 +12,9 @@ CreateScorpioTargets() include (${SCREAM_BASE_DIR}/cmake/tpls/CsmShare.cmake) CreateCsmShareTarget() -if (SCREAM_CIME_BUILD) - # For CIME runs, wrap mct in a target too - include (${SCREAM_BASE_DIR}/cmake/tpls/Mct.cmake) - CreateMctTarget() +if (NOT SCREAM_LIB_ONLY) + include(BuildCprnc) + BuildCprnc() endif() # MAM aerosol support @@ -22,6 +22,22 @@ if (SCREAM_ENABLE_MAM) # We use CMake's ExternalProject capability to build and install Haero. include(ExternalProject) + if (SCREAM_CIME_BUILD) + # PROJECT_SOURCE_DIR is SCREAM_ROOT/components + set(EXTERNALS_DIR "${PROJECT_SOURCE_DIR}/../externals") + else() + # PROJECT_SOURCE_DIR is SCREAM_ROOT/components/eamxx + set(EXTERNALS_DIR "${PROJECT_SOURCE_DIR}/../../externals") + endif() + + # Normalize CMAKE_BUILD_TYPE. + string(TOLOWER "${CMAKE_BUILD_TYPE}" lc_build_type) + if(${lc_build_type} STREQUAL "release") + set(mam_build_type "Release") + else() + set(mam_build_type "Debug") # when in doubt... + endif() + # Build and install the Haero aerosol package interface. if (SCREAM_DOUBLE_PRECISION) set(HAERO_PRECISION "double") @@ -31,16 +47,20 @@ if (SCREAM_ENABLE_MAM) set(HAERO_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/externals/haero") set(HAERO_CMAKE_OPTS -DCMAKE_INSTALL_PREFIX=${HAERO_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_BUILD_TYPE=${mam_build_type} -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DHAERO_ENABLE_GPU=${Kokkos_ENABLE_CUDA} -DHAERO_ENABLE_MPI=ON -DHAERO_PRECISION=${HAERO_PRECISION} - -DEKAT_SOURCE_DIR=${PROJECT_SOURCE_DIR}/../../externals/ekat - -DEKAT_BINARY_DIR=${PROJECT_BINARY_DIR}/externals/ekat) + -DEKAT_SOURCE_DIR=${EXTERNALS_DIR}/ekat + -DEKAT_BINARY_DIR=${PROJECT_BINARY_DIR}/externals/ekat + -DBUILD_SHARED_LIBS=OFF) ExternalProject_Add(haero_proj PREFIX ${PROJECT_BINARY_DIR}/externals/haero - SOURCE_DIR ${PROJECT_SOURCE_DIR}/../../externals/haero + SOURCE_DIR ${EXTERNALS_DIR}/haero BINARY_DIR ${PROJECT_BINARY_DIR}/externals/haero CMAKE_ARGS ${HAERO_CMAKE_OPTS} DEPENDS ekat ekat_test_session ekat_test_main @@ -48,18 +68,24 @@ if (SCREAM_ENABLE_MAM) BUILD_COMMAND make -j LOG_BUILD TRUE INSTALL_COMMAND make install - LOG_INSTALL TRUE) + LOG_INSTALL TRUE + LOG_OUTPUT_ON_FAILURE TRUE + BUILD_ALWAYS TRUE) + add_library(haero STATIC IMPORTED GLOBAL) + set_target_properties(haero PROPERTIES IMPORTED_LOCATION ${HAERO_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libhaero.a) # Build and install MAM4xx. set(MAM4XX_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/externals/mam4xx") set(MAM4XX_CMAKE_OPTS -DCMAKE_INSTALL_PREFIX=${MAM4XX_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_BUILD_TYPE=${mam_build_type} -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE} - -DMAM4XX_HAERO_DIR=${HAERO_INSTALL_PREFIX}) + -DMAM4XX_HAERO_DIR=${HAERO_INSTALL_PREFIX} + -DBUILD_SHARED_LIBS=OFF + ) ExternalProject_Add(mam4xx_proj PREFIX ${PROJECT_BINARY_DIR}/externals/mam4xx - SOURCE_DIR ../../../../externals/mam4xx + SOURCE_DIR ${EXTERNALS_DIR}/mam4xx BINARY_DIR ${PROJECT_BINARY_DIR}/externals/mam4xx CMAKE_ARGS ${MAM4XX_CMAKE_OPTS} DEPENDS haero_proj @@ -67,8 +93,11 @@ if (SCREAM_ENABLE_MAM) BUILD_COMMAND make -j LOG_BUILD TRUE INSTALL_COMMAND make install - LOG_INSTALL TRUE) - + LOG_INSTALL TRUE + LOG_OUTPUT_ON_FAILURE TRUE + BUILD_ALWAYS TRUE) + add_library(mam4xx STATIC IMPORTED GLOBAL) + set_target_properties(mam4xx PROPERTIES IMPORTED_LOCATION ${MAM4XX_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libmam4xx.a) # Bring in MAM4xx-related targets by including the generated mam4xx.cmake # list(APPEND CMAKE_MODULE_PATH ${MAM4XX_INSTALL_PREFIX}/share) # include(mam4xx) diff --git a/components/elm/cime_config/config_component.xml b/components/elm/cime_config/config_component.xml index ea02f3e93538..b625542add53 100755 --- a/components/elm/cime_config/config_component.xml +++ b/components/elm/cime_config/config_component.xml @@ -87,6 +87,7 @@ 2010_CMIP6HR_control 2010_defscream_control + 2010_defscream_control 2010_dy2scream_control 2000_control diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index fe6aafb6ec34..c016182e3bf0 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -456,9 +456,9 @@ IF (HOMME_USE_KOKKOS AND HOMME_STANDALONE) list(APPEND CMAKE_MODULE_PATH ${EKAT_CMAKE_PATH} ${EKAT_CMAKE_PATH}/pkg_build + ${EKAT_CMAKE_PATH}/tpls ) include (EkatBuildKokkos) - BuildKokkos() ENDIF () # This folder contains the CMake macro used to build cxx unit tests diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 8595988bf23c..6d073dbbe83b 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -156,7 +156,7 @@ macro(createTestExec execName execType macroNP macroNC ENDIF () IF (HOMME_USE_KOKKOS) - target_link_libraries(${execName} kokkos) + target_link_libraries(${execName} Kokkos::kokkos) ENDIF () # Move the module files out of the way so the parallel build @@ -260,7 +260,7 @@ macro(createExecLib libName execType libSrcs inclDirs macroNP TARGET_LINK_LIBRARIES(${libName} timing ${COMPOSE_LIBRARY} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_KOKKOS) - TARGET_LINK_LIBRARIES(${libName} kokkos) + TARGET_LINK_LIBRARIES(${libName} Kokkos::kokkos) ENDIF () IF (HOMME_USE_MKL) diff --git a/components/homme/src/share/compose/CMakeLists.txt b/components/homme/src/share/compose/CMakeLists.txt index 3e22fb6fdc84..a052dcc30326 100644 --- a/components/homme/src/share/compose/CMakeLists.txt +++ b/components/homme/src/share/compose/CMakeLists.txt @@ -45,4 +45,4 @@ add_library (${COMPOSE_LIBRARY} cedr_qlt.cpp cedr_test_randomized.cpp) -target_link_libraries(${COMPOSE_LIBRARY} kokkos) +target_link_libraries(${COMPOSE_LIBRARY} Kokkos::kokkos) diff --git a/components/homme/test/unit_tests/CMakeLists.txt b/components/homme/test/unit_tests/CMakeLists.txt index 23c6b1a3169d..2b601ae172cf 100644 --- a/components/homme/test/unit_tests/CMakeLists.txt +++ b/components/homme/test/unit_tests/CMakeLists.txt @@ -22,7 +22,7 @@ macro(cxx_unit_test target_name target_f90_srcs target_cxx_srcs include_dirs con cxx_unit_test_add_test(${target_name}_test ${target_name} ${NUM_CPUS}) TARGET_LINK_LIBRARIES(${target_name} timing ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) - target_link_libraries(${target_name} kokkos) + target_link_libraries(${target_name} Kokkos::kokkos) IF (HOMME_USE_MKL) TARGET_COMPILE_OPTIONS (${target_name} PUBLIC -mkl) TARGET_LINK_LIBRARIES (${target_name} -mkl) diff --git a/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt index cbbe129100bc..9dd30f58f63f 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt @@ -36,7 +36,7 @@ ADD_LIBRARY(preqx_kokkos_ut_lib TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS}) TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) TARGET_COMPILE_DEFINITIONS(preqx_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H") -target_link_libraries(preqx_kokkos_ut_lib kokkos) +target_link_libraries(preqx_kokkos_ut_lib Kokkos::kokkos) TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib timing csm_share ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_MKL) TARGET_LINK_LIBRARIES (preqx_kokkos_ut_lib -mkl) diff --git a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt index 6e3b5080aecd..205635e918cc 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt @@ -39,7 +39,7 @@ ADD_LIBRARY(thetal_kokkos_ut_lib TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS}) TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H") -target_link_libraries(thetal_kokkos_ut_lib kokkos) +target_link_libraries(thetal_kokkos_ut_lib Kokkos::kokkos) TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib timing csm_share ${COMPOSE_LIBRARY_CPP} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_MKL) TARGET_LINK_LIBRARIES (thetal_kokkos_ut_lib -mkl) diff --git a/docs/index.md b/docs/index.md index 9bd2de54b2a6..36584c99bf7c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,7 +3,8 @@ The documentation for the components of E3SM is found here. ## Components -- [ELM](./ELM/index.md) - [EAM](./EAM/index.md) +- [EAMxx](./EAMxx/index.md) +- [ELM](./ELM/index.md) - [MOSART](./MOSART/index.md) diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index dd7a5ddb6c15..5d582b41b438 100755 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -432,6 +432,7 @@ 1 $ATM_NCPL 48 + $ATM_NCPL $ATM_NCPL 12 96 diff --git a/externals/YAKL b/externals/YAKL index a032296858f9..4109dc02fe1e 160000 --- a/externals/YAKL +++ b/externals/YAKL @@ -1 +1 @@ -Subproject commit a032296858f9069b2ca61243802655b607fab0a0 +Subproject commit 4109dc02fe1e951a95f401719587e981ea4f4fe4 diff --git a/externals/ekat b/externals/ekat index 84125bc7381b..2ff5853316e1 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 84125bc7381b999834e303cea149629943a55cad +Subproject commit 2ff5853316e15d4e8004c21890329fd257fa7459 diff --git a/externals/haero b/externals/haero index c66390cdf149..574787f9b31c 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit c66390cdf14947b5b039e926f5fead667e2cc45a +Subproject commit 574787f9b31cc77b7902f07e73e10dcd18cdf29d diff --git a/externals/mam4xx b/externals/mam4xx index 1eca842c22b4..8f6af492c862 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 1eca842c22b441a4ceedb7e772bcccdd1735f3d0 +Subproject commit 8f6af492c8627ebe5ec7904c3185ab8294e49a12 diff --git a/share/CMakeLists.txt b/share/CMakeLists.txt index f7dcffcbb171..1112f9580e3b 100644 --- a/share/CMakeLists.txt +++ b/share/CMakeLists.txt @@ -29,32 +29,12 @@ function(set_compilers_e3sm) set(CMAKE_CXX_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "The CXX compiler") set(CMAKE_C_COMPILER ${CMAKE_C_COMPILER} CACHE STRING "The C compiler") set(CMAKE_Fortran_COMPILER ${CMAKE_Fortran_COMPILER} CACHE STRING "The Fortran compiler") - - # USE_CUDA or USE_HIP is set through Macros.cmake - # For instance: cime_config/machines/cmake_macros/gnugpu_summit.cmake - # If it exists, then set parent's scope to true; otherwise to false - # At this point, we use either CUDA or HIP. - # Revisit as needed for future systems. - if (USE_CUDA) - set(USE_CUDA TRUE PARENT_SCOPE) - elseif (USE_HIP) - set(USE_HIP TRUE PARENT_SCOPE) - else() - set(USE_CUDA FALSE PARENT_SCOPE) - set(USE_HIP FALSE PARENT_SCOPE) - endif() endfunction() set_compilers_e3sm() project(CSM_SHARE C CXX Fortran) -if(USE_CUDA) - enable_language(CUDA) -elseif(USE_HIP) - enable_language(HIP) -endif() - # Any changes to SourceMods will require us to reconfigure file(GLOB COMPONENT_SOURCE_MOD_DIRS "${CASEROOT}/SourceMods/src.*") foreach(COMPONENT_SOURCE_MOD_DIR IN LISTS COMPONENT_SOURCE_MOD_DIRS)