From f6246e12226bd0a933f218eeaddfe33d4039e323 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 4 Nov 2023 17:46:38 -0700 Subject: [PATCH 001/142] Started with MAMOptics and changed to MAMAci --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- .../mam/eamxx_mam_aci_process_interface.cpp | 107 ++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 81 +++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 53eb4049f11..cfd33e88d0b 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -42,7 +42,8 @@ add_subdirectory(${EXTERNALS_SOURCE_DIR}/mam4xx ${CMAKE_BINARY_DIR}/externals/ma # EAMxx mam4xx-based atmospheric processes add_library(mam eamxx_mam_microphysics_process_interface.cpp - eamxx_mam_optics_process_interface.cpp) + eamxx_mam_optics_process_interface.cpp + eamxx_mam_aci_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp new file mode 100644 index 00000000000..fc27eafcf9d --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -0,0 +1,107 @@ +#include +#include +#include + +#include "scream_config.h" // for SCREAM_CIME_BUILD + +#include + +namespace scream +{ + +MAMAci::MAMAci( + const ekat::Comm& comm, + const ekat::ParameterList& params) + : AtmosphereProcess(comm, params), + aero_config_() { +} + +AtmosphereProcessType MAMAci::type() const { + return AtmosphereProcessType::Physics; +} + +std::string MAMAci::name() const { + return "mam4_optics"; +} + +void MAMAci::set_grids(const std::shared_ptr grids_manager) { + using namespace ekat::units; + + grid_ = grids_manager->get_grid("Physics"); + const auto& grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // number of levels per column + nswbands_ = 14; // number of shortwave bands + nlwbands_ = 16; // number of longwave bands + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Define aerosol optics fields computed by this process. + auto nondim = Units::nondimensional(); + FieldLayout scalar3d_swband_layout { {COL, SWBND, LEV}, {ncol_, nswbands_, nlev_} }; + FieldLayout scalar3d_lwband_layout { {COL, LWBND, LEV}, {ncol_, nlwbands_, nlev_} }; + + // shortwave aerosol scattering asymmetry parameter [-] + add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); + // shortwave aerosol single-scattering albedo [-] + add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); + // shortwave aerosol optical depth [-] + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + // longwave aerosol optical depth [-] + add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); + + // FIXME: this field doesn't belong here, but this is a convenient place to + // FIXME: put it for now. + // number mixing ratio for CCN + using Spack = ekat::Pack; + using Pack = ekat::Pack; + constexpr int ps = Pack::n; + FieldLayout scalar3d_layout_mid { {COL, LEV}, {ncol_, nlev_} }; + add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); +} + +void MAMAci::initialize_impl(const RunType run_type) { +} + +void MAMAci::run_impl(const double dt) { + + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + + // get the aerosol optics fields + auto aero_g_sw = get_field_out("aero_g_sw").get_view(); + auto aero_ssa_sw = get_field_out("aero_ssa_sw").get_view(); + auto aero_tau_sw = get_field_out("aero_tau_sw").get_view(); + auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); + + auto aero_nccn = get_field_out("nccn").get_view(); // FIXME: get rid of this + + // Compute optical properties on all local columns. + // (Strictly speaking, we don't need this parallel_for here yet, but we leave + // it in anticipation of column-specific aerosol optics to come.) + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { + const Int icol = team.league_rank(); // column index + + auto g_sw = ekat::subview(aero_g_sw, icol); + auto ssa_sw = ekat::subview(aero_ssa_sw, icol); + auto tau_sw = ekat::subview(aero_tau_sw, icol); + auto tau_lw = ekat::subview(aero_tau_lw, icol); + + // populate these fields with reasonable representative values + Kokkos::deep_copy(g_sw, 0.5); + Kokkos::deep_copy(ssa_sw, 0.7); + Kokkos::deep_copy(tau_sw, 0.0); + Kokkos::deep_copy(tau_lw, 0.0); + + // FIXME: Get rid of this + auto nccn = ekat::subview(aero_nccn, icol); + Kokkos::deep_copy(nccn, 50.0); + }); +} + +void MAMAci::finalize_impl() +{ +} + +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp new file mode 100644 index 00000000000..0dbcbf75366 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -0,0 +1,81 @@ +#ifndef EAMXX_MAM_ACI_HPP +#define EAMXX_MAM_ACI_HPP + +#include +#include +#include + +#include +#include +#include + +#include + +#ifndef KOKKOS_ENABLE_CUDA +#define protected_except_cuda public +#define private_except_cuda public +#else +#define protected_except_cuda protected +#define private_except_cuda private +#endif + +namespace scream +{ + +// The process responsible for handling MAM4 aerosol optical properties. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMAci final : public scream::AtmosphereProcess { + using PF = scream::PhysicsFunctions; + using KT = ekat::KokkosTypes; + + // a quantity stored in a single vertical column with a single index + using ColumnView = mam4::ColumnView; + + // a thread team dispatched to a single vertical column + using ThreadTeam = mam4::ThreadTeam; + +public: + + // Constructor + MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); + +protected_except_cuda: + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // process metadata + AtmosphereProcessType type() const override; + std::string name() const override; + + // grid + void set_grids(const std::shared_ptr grids_manager) override; + + // process behavior + void initialize_impl(const RunType run_type) override; + void run_impl(const double dt) override; + void finalize_impl() override; + +private_except_cuda: + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // number of shortwave and longwave radiation bands + int nswbands_, nlwbands_; + + // MAM4 aerosol particle size description + mam4::AeroConfig aero_config_; + + // aerosol processes + //std::unique_ptr optics_; + + // physics grid for column information + std::shared_ptr grid_; +}; // MAMAci + +} // namespace scream + + +#endif // EAMXX_MAM_ACI_HPP From 3da2c97b7b347787a9f5bf767ad4e5b60e591492 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 6 Nov 2023 11:09:50 -0800 Subject: [PATCH 002/142] Minimal additions for a MAMAci class with a constructor --- .../mam/eamxx_mam_aci_process_interface.cpp | 97 +------------------ .../mam/eamxx_mam_aci_process_interface.hpp | 62 ------------ 2 files changed, 1 insertion(+), 158 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index fc27eafcf9d..e550ad55fe7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,10 +1,4 @@ #include -#include -#include - -#include "scream_config.h" // for SCREAM_CIME_BUILD - -#include namespace scream { @@ -12,96 +6,7 @@ namespace scream MAMAci::MAMAci( const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereProcess(comm, params), - aero_config_() { -} - -AtmosphereProcessType MAMAci::type() const { - return AtmosphereProcessType::Physics; -} - -std::string MAMAci::name() const { - return "mam4_optics"; -} - -void MAMAci::set_grids(const std::shared_ptr grids_manager) { - using namespace ekat::units; - - grid_ = grids_manager->get_grid("Physics"); - const auto& grid_name = grid_->name(); - - ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // number of levels per column - nswbands_ = 14; // number of shortwave bands - nlwbands_ = 16; // number of longwave bands - - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; - - // Define aerosol optics fields computed by this process. - auto nondim = Units::nondimensional(); - FieldLayout scalar3d_swband_layout { {COL, SWBND, LEV}, {ncol_, nswbands_, nlev_} }; - FieldLayout scalar3d_lwband_layout { {COL, LWBND, LEV}, {ncol_, nlwbands_, nlev_} }; - - // shortwave aerosol scattering asymmetry parameter [-] - add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); - // shortwave aerosol single-scattering albedo [-] - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); - // shortwave aerosol optical depth [-] - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); - // longwave aerosol optical depth [-] - add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); - - // FIXME: this field doesn't belong here, but this is a convenient place to - // FIXME: put it for now. - // number mixing ratio for CCN - using Spack = ekat::Pack; - using Pack = ekat::Pack; - constexpr int ps = Pack::n; - FieldLayout scalar3d_layout_mid { {COL, LEV}, {ncol_, nlev_} }; - add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); -} - -void MAMAci::initialize_impl(const RunType run_type) { -} - -void MAMAci::run_impl(const double dt) { - - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - - // get the aerosol optics fields - auto aero_g_sw = get_field_out("aero_g_sw").get_view(); - auto aero_ssa_sw = get_field_out("aero_ssa_sw").get_view(); - auto aero_tau_sw = get_field_out("aero_tau_sw").get_view(); - auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); - - auto aero_nccn = get_field_out("nccn").get_view(); // FIXME: get rid of this - - // Compute optical properties on all local columns. - // (Strictly speaking, we don't need this parallel_for here yet, but we leave - // it in anticipation of column-specific aerosol optics to come.) - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { - const Int icol = team.league_rank(); // column index - - auto g_sw = ekat::subview(aero_g_sw, icol); - auto ssa_sw = ekat::subview(aero_ssa_sw, icol); - auto tau_sw = ekat::subview(aero_tau_sw, icol); - auto tau_lw = ekat::subview(aero_tau_lw, icol); - - // populate these fields with reasonable representative values - Kokkos::deep_copy(g_sw, 0.5); - Kokkos::deep_copy(ssa_sw, 0.7); - Kokkos::deep_copy(tau_sw, 0.0); - Kokkos::deep_copy(tau_lw, 0.0); - - // FIXME: Get rid of this - auto nccn = ekat::subview(aero_nccn, icol); - Kokkos::deep_copy(nccn, 50.0); - }); -} - -void MAMAci::finalize_impl() -{ + : AtmosphereProcess(comm, params){ } } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 0dbcbf75366..e54c2e1400a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -2,77 +2,15 @@ #define EAMXX_MAM_ACI_HPP #include -#include -#include - -#include -#include -#include - -#include - -#ifndef KOKKOS_ENABLE_CUDA -#define protected_except_cuda public -#define private_except_cuda public -#else -#define protected_except_cuda protected -#define private_except_cuda private -#endif namespace scream { -// The process responsible for handling MAM4 aerosol optical properties. The AD -// stores exactly ONE instance of this class in its list of subcomponents. class MAMAci final : public scream::AtmosphereProcess { - using PF = scream::PhysicsFunctions; - using KT = ekat::KokkosTypes; - - // a quantity stored in a single vertical column with a single index - using ColumnView = mam4::ColumnView; - - // a thread team dispatched to a single vertical column - using ThreadTeam = mam4::ThreadTeam; public: - // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); - -protected_except_cuda: - - // -------------------------------------------------------------------------- - // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) - // -------------------------------------------------------------------------- - - // process metadata - AtmosphereProcessType type() const override; - std::string name() const override; - - // grid - void set_grids(const std::shared_ptr grids_manager) override; - - // process behavior - void initialize_impl(const RunType run_type) override; - void run_impl(const double dt) override; - void finalize_impl() override; - -private_except_cuda: - - // number of horizontal columns and vertical levels - int ncol_, nlev_; - - // number of shortwave and longwave radiation bands - int nswbands_, nlwbands_; - - // MAM4 aerosol particle size description - mam4::AeroConfig aero_config_; - - // aerosol processes - //std::unique_ptr optics_; - - // physics grid for column information - std::shared_ptr grid_; }; // MAMAci } // namespace scream From 13861452a9187458405d80197802286a98cdb0bb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 7 Nov 2023 14:59:50 -0800 Subject: [PATCH 003/142] Model compiles after registering the process, added extra stuff, will remove in the next commit --- .../mam/eamxx_mam_aci_process_interface.cpp | 29 +++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 23 +++++++++++++++ .../eamxx/src/physics/register_physics.hpp | 2 ++ 3 files changed, 54 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e550ad55fe7..45824480728 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,4 +1,11 @@ #include +#include "scream_config.h" // for SCREAM_CIME_BUILD +#include +#include + +#include "scream_config.h" // for SCREAM_CIME_BUILD + +#include namespace scream { @@ -9,4 +16,26 @@ MAMAci::MAMAci( : AtmosphereProcess(comm, params){ } +AtmosphereProcessType MAMAci::type() const { + return AtmosphereProcessType::Physics; +} +//return name of the process +std::string MAMAci::name() const{ + return "mam4_aci"; + } + + +//grid +void MAMAci::set_grids(const std::shared_ptr grids_manager) { +} + +void MAMAci::initialize_impl(const RunType run_type) { +} + +void MAMAci::run_impl(const double dt) { +} + +void MAMAci::finalize_impl(){ +} + } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index e54c2e1400a..ca89537db96 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -2,6 +2,14 @@ #define EAMXX_MAM_ACI_HPP #include +#include +#include + +#include +#include +#include + +#include namespace scream { @@ -11,8 +19,23 @@ class MAMAci final : public scream::AtmosphereProcess { public: // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); +// process metadata + AtmosphereProcessType type() const override; + std::string name() const override; + + // grid + void set_grids(const std::shared_ptr grids_manager) override; + +// process behavior + void initialize_impl(const RunType run_type) override; + void run_impl(const double dt) override; + void finalize_impl() override; + + + }; // MAMAci + } // namespace scream diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 329b7022756..36afabf8de9 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -26,6 +26,7 @@ #ifdef EAMXX_HAS_MAM #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" +#include "physics/mam/eamxx_mam_aci_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -62,6 +63,7 @@ inline void register_physics () { #ifdef EAMXX_HAS_MAM proc_factory.register_product("mam4_micro",&create_atmosphere_process); proc_factory.register_product("mam4_optics",&create_atmosphere_process); + proc_factory.register_product("mam4_aci",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 10f5790efdf84f6cd0f81a95fa294de4bbbcf572 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 8 Nov 2023 08:19:08 -0800 Subject: [PATCH 004/142] Removed un-needed code, minimal code for invoking and printing init,set grid, run and finalize aci process --- .../mam/eamxx_mam_aci_process_interface.cpp | 20 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 16 ++------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 45824480728..b753e3d7ee8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,41 +1,39 @@ #include -#include "scream_config.h" // for SCREAM_CIME_BUILD -#include -#include - -#include "scream_config.h" // for SCREAM_CIME_BUILD - -#include namespace scream { - MAMAci::MAMAci( const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params){ } + +//Return type of the process AtmosphereProcessType MAMAci::type() const { return AtmosphereProcessType::Physics; } + //return name of the process std::string MAMAci::name() const{ return "mam4_aci"; } - -//grid +//set grid for all the inputs and outputs void MAMAci::set_grids(const std::shared_ptr grids_manager) { + m_atm_logger->log(ekat::logger::LogLevel::info,"ACI set grid = "); } void MAMAci::initialize_impl(const RunType run_type) { + m_atm_logger->log(ekat::logger::LogLevel::info,"ACI init = "); } void MAMAci::run_impl(const double dt) { + m_atm_logger->log(ekat::logger::LogLevel::info,"ACI run = "); } void MAMAci::finalize_impl(){ + m_atm_logger->log(ekat::logger::LogLevel::info,"ACI final = "); } -} // namespace scream +} // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index ca89537db96..0d361ba9f41 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -2,14 +2,6 @@ #define EAMXX_MAM_ACI_HPP #include -#include -#include - -#include -#include -#include - -#include namespace scream { @@ -19,23 +11,19 @@ class MAMAci final : public scream::AtmosphereProcess { public: // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); -// process metadata + // process metadata AtmosphereProcessType type() const override; std::string name() const override; // grid void set_grids(const std::shared_ptr grids_manager) override; -// process behavior + // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; void finalize_impl() override; - - - }; // MAMAci - } // namespace scream From 4a59acad24d55aec3f81996f2fcae3f9342db347 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 14 Nov 2023 08:29:34 -0800 Subject: [PATCH 005/142] Minimal code for adding a test for aci with two outputs --- .../mam/eamxx_mam_aci_process_interface.cpp | 17 +++++ .../mam/eamxx_mam_aci_process_interface.hpp | 9 +++ .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 34 +++++++++ .../tests/uncoupled/mam4/mam4_aci_output.yaml | 12 ++++ .../uncoupled/mam4/mam4_aci_standalone.cpp | 69 +++++++++++++++++++ 5 files changed, 141 insertions(+) create mode 100644 components/eamxx/tests/uncoupled/mam4/input_aci.yaml create mode 100644 components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml create mode 100644 components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b753e3d7ee8..c72ddfd43e0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -22,10 +22,27 @@ std::string MAMAci::name() const{ //set grid for all the inputs and outputs void MAMAci::set_grids(const std::shared_ptr grids_manager) { m_atm_logger->log(ekat::logger::LogLevel::info,"ACI set grid = "); + + grid_ = grids_manager->get_grid("Physics"); + const auto& grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variables + FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; + + using namespace ekat::units; + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // total pressure } void MAMAci::initialize_impl(const RunType run_type) { m_atm_logger->log(ekat::logger::LogLevel::info,"ACI init = "); + std::cout<<"BALLI==============================="< grid_; }; // MAMAci } // namespace scream diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml new file mode 100644 index 00000000000..b254d95a735 --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -0,0 +1,34 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_aci] + mam4_micro: + compute_tendencies: [] + +grids_manager: + Type: Mesh Free + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + T_mid: 273.0 + p_mid: 1.e5 + qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["mam4_aci_output.yaml"] +... diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml b/components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml new file mode 100644 index 00000000000..0087223d1d6 --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml @@ -0,0 +1,12 @@ +%YAML 1.1 +--- +filename_prefix: mam4_aci_standalone_output +Averaging Type: Instant +Field Names: + - T_mid + - p_mid +output_control: + Frequency: 1 + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp new file mode 100644 index 00000000000..8100fd37a1f --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp @@ -0,0 +1,69 @@ +#include + +#include "control/atmosphere_driver.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "physics/register_physics.hpp" +#include "physics/mam/eamxx_mam_aci_process_interface.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "ekat/ekat_parse_yaml_file.hpp" +#include "ekat/logging/ekat_logger.hpp" + +#include + +namespace scream { + +TEST_CASE("mam4-aci-standalone", "") { + using namespace scream; + using namespace scream::control; + + // Create a comm + ekat::Comm atm_comm (MPI_COMM_WORLD); + ekat::logger::Logger<> logger("mam4-aci", + ekat::logger::LogLevel::debug, atm_comm); + + + // Load ad parameter list + std::string fname = "input_aci.yaml"; + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + logger.debug("aci yaml parsed."); + + // Time stepping parameters + const auto& ts = ad_params.sublist("time_stepping"); + const auto dt = ts.get("time_step"); + const auto nsteps = ts.get("number_of_steps"); + const auto t0_str = ts.get("run_t0"); + const auto t0 = util::str_to_time_stamp(t0_str); + + logger.info("running MAMAci standalone test with dt = {} for {} steps.", dt, nsteps); + + // Need to register products in the factory *before* we create any atm process or grids manager. + register_physics(); + register_mesh_free_grids_manager(); + register_diagnostics(); + logger.debug("products registered."); + + // Create the driver + AtmosphereDriver ad; + logger.debug("driver created."); + + // Init and run + ad.initialize(atm_comm,ad_params,t0); + logger.debug("driver initialized."); + + logger.info("Start time stepping loop ... [0%]"); + for (int i=0; i Date: Tue, 14 Nov 2023 15:03:35 -0800 Subject: [PATCH 006/142] Adds some required inputs for the aci process --- .../mam/eamxx_mam_aci_process_interface.cpp | 34 +++++++++++++++++-- .../uncoupled/mam4/mam4_aci_standalone.cpp | 1 - 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index c72ddfd43e0..808dfc75d83 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -33,11 +33,39 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) using namespace ShortFieldTagsNames; // Layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; + FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points + FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces using namespace ekat::units; - add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // total pressure + auto q_unit = kg/kg; // units of mass mixing ratios of tracers + auto n_unit = kg/kg; // units of number mixing ratios of tracers + + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid mass mixing ratio [kg/kg] + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints + add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints + add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios + /*for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + } + } + }*/ + /*NOTE on other inputs for the aci process: + 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density + 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using + pseudo_density, p_mid, T_mid and qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). + qv_mid can be obtained from "get_field_in" call*/ + } void MAMAci::initialize_impl(const RunType run_type) { diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp index 8100fd37a1f..dd3ed62ec15 100644 --- a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp +++ b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp @@ -24,7 +24,6 @@ TEST_CASE("mam4-aci-standalone", "") { ekat::logger::Logger<> logger("mam4-aci", ekat::logger::LogLevel::debug, atm_comm); - // Load ad parameter list std::string fname = "input_aci.yaml"; ekat::ParameterList ad_params("Atmosphere Driver"); From 3dad80470927ca400560c69322c80842924d557e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 14 Nov 2023 15:56:02 -0800 Subject: [PATCH 007/142] Fixes unit for the number mixing ratio --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 808dfc75d83..52a282ab569 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -38,7 +38,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) using namespace ekat::units; auto q_unit = kg/kg; // units of mass mixing ratios of tracers - auto n_unit = kg/kg; // units of number mixing ratios of tracers + auto n_unit = 1/kg; // units of number mixing ratios of tracers add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] From 1557547f53ae273ebce8f1b7f981a58a1c6494a3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 14 Nov 2023 18:08:30 -0800 Subject: [PATCH 008/142] Adds Commented out code to stage next changes --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index dc788898073..35774059850 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -1,6 +1,8 @@ #ifndef EAMXX_MAM_ACI_HPP #define EAMXX_MAM_ACI_HPP +//#include +//#include #include namespace scream From f9539b1b3b4993274b1845d07baec5a650f661a8 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 17 Nov 2023 14:32:08 -0800 Subject: [PATCH 009/142] Adds aerosol fields as required and also in the input yaml --- .../mam/eamxx_mam_aci_process_interface.cpp | 7 +++--- .../mam/eamxx_mam_aci_process_interface.hpp | 3 +-- .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 25 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 52a282ab569..cae35dc2b93 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -50,7 +50,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios - /*for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { @@ -59,7 +59,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } - }*/ + } + /*NOTE on other inputs for the aci process: 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using @@ -81,4 +82,4 @@ void MAMAci::finalize_impl(){ m_atm_logger->log(ekat::logger::LogLevel::info,"ACI final = "); } -} // namespace scream \ No newline at end of file +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 35774059850..07e553ce757 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -1,8 +1,7 @@ #ifndef EAMXX_MAM_ACI_HPP #define EAMXX_MAM_ACI_HPP -//#include -//#include +#include #include namespace scream diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index b254d95a735..7927b241835 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -26,6 +26,31 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} T_mid: 273.0 p_mid: 1.e5 + num_a1: 1.e5 + bc_a1: 1.e-5 + num_a2: 1e5 + num_a3: 1e5 + num_a4: 1e5 + soa_a1: 1e-5 + so4_a1: 1e-5 + pom_a1: 1e-5 + nacl_a1: 1e-5 + dst_a1: 1e-5 + mom_a1: 1e-5 + soa_a2: 1e-5 + so4_a2: 1e-5 + nacl_a2: 1e-5 + mom_a2: 1e-5 + soa_a3: 1e-5 + so4_a3: 1e-5 + pom_a3: 1e-5 + bc_a3: 1e-5 + nacl_a3: 1e-5 + dst_a3: 1e-5 + mom_a3: 1e-5 + pom_a4: 1e-5 + bc_a4: 1e-5 + mom_a4: 1e-5 qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae # The parameters for I/O control From 13333316db6e3d577e578a98b927858bd381b3fc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 17 Nov 2023 15:13:39 -0800 Subject: [PATCH 010/142] Adds cloud borne aerosols; fixes a minor bug in mam coupling --- .../mam/eamxx_mam_aci_process_interface.cpp | 8 +++++++ .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 22 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index cae35dc2b93..f848e6dd8f5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -59,6 +59,14 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } + for (int c = 0; c < mam_coupling::num_aero_species(); ++c) { + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, c); + std::cout< 0) { + //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); + } + } } /*NOTE on other inputs for the aci process: diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index 7927b241835..3704235bdf9 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -51,6 +51,28 @@ initial_conditions: pom_a4: 1e-5 bc_a4: 1e-5 mom_a4: 1e-5 + #cloud borne aerosols + soa_c1: 1e-5 + so4_c1: 1e-5 + pom_c1: 1e-5 + bc_c1: 1e-5 + nacl_c1: 1e-5 + dst_c1: 1e-5 + mom_c1: 1e-5 + soa_c2: 1e-5 + so4_c2: 1e-5 + nacl_c2: 1e-5 + mom_c2: 1e-5 + soa_c3: 1e-5 + so4_c3: 1e-5 + pom_c3: 1e-5 + bc_c3: 1e-5 + nacl_c3: 1e-5 + dst_c3: 1e-5 + mom_c3: 1e-5 + pom_c4: 1e-5 + bc_c4: 1e-5 + mom_c4: 1e-5 qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae # The parameters for I/O control From 25d45f0820475a29f0fb79e82588531a5c4e7c61 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 20 Nov 2023 19:43:01 -0800 Subject: [PATCH 011/142] Adds all input output variables in set grid --- .../mam/eamxx_mam_aci_process_interface.cpp | 21 +++++++++++++++++-- .../mam/eamxx_mam_aci_process_interface.hpp | 4 ++++ .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 6 +++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index f848e6dd8f5..9818160f914 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -37,8 +37,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces using namespace ekat::units; - auto q_unit = kg/kg; // units of mass mixing ratios of tracers - auto n_unit = 1/kg; // units of number mixing ratios of tracers + const auto q_unit = kg/kg; // units of mass mixing ratios of tracers + const auto n_unit = 1/kg; // units of number mixing ratios of tracers add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] @@ -69,6 +69,23 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) } } + //input for aci codes that existed in PBUF in EAM + //These outputs should come from the cloud macrophysics process (e.g., SHOC) + const auto m2 = m*m; + const auto s2 = s*s; + //FIXME BALLI: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time + add_field("w_sec", scalar3d_layout_mid, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints + + auto nondim = Units::nondimensional(); + add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints + add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints + add_field("kvh", scalar3d_layout_mid, m2/s, grid_name); // Eddy diffusivity for heat + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_modes_} }; // mid points + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols + + /*NOTE on other inputs for the aci process: 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 07e553ce757..232bcf95abc 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -30,6 +30,10 @@ class MAMAci final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; + // number of aerosol modes + int num_modes_; + + // physics grid for column information std::shared_ptr grid_; }; // MAMAci diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index 3704235bdf9..a14ec51d036 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -74,7 +74,11 @@ initial_conditions: bc_c4: 1e-5 mom_c4: 1e-5 qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae - + w_sec: 1e-2 + strat_cld_frac: 0.1 + liq_strat_cld_frac: 0.1 + kvh: 0.1 + dgnum: 1e-3 # The parameters for I/O control Scorpio: output_yaml_files: ["mam4_aci_output.yaml"] From 7f9d5a3db5bf4bc3d884caeb2904553e50b9ade5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 25 Nov 2023 10:58:42 -0800 Subject: [PATCH 012/142] Adds commented out Fortran code with comments, some cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 209 +++++++++++++++++- .../mam/eamxx_mam_aci_process_interface.hpp | 8 +- .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 4 + 3 files changed, 208 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 9818160f914..7e35ef92313 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -2,6 +2,10 @@ namespace scream { + + //FIXME: The following variables are namelist variables + Real wsubmin = 1; + MAMAci::MAMAci( const ekat::Comm& comm, const ekat::ParameterList& params) @@ -37,9 +41,12 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces using namespace ekat::units; - const auto q_unit = kg/kg; // units of mass mixing ratios of tracers - const auto n_unit = 1/kg; // units of number mixing ratios of tracers + auto q_unit = kg/kg; // units of mass mixing ratios of tracers + q_unit.set_string("kg/kg"); + auto n_unit = 1/kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + // atmospheric quantities add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid mass mixing ratio [kg/kg] @@ -49,19 +56,27 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios + // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + //interstitial aerosol tracers of interest: number (n) mixing ratios const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + //cloudborne aerosol tracers of interest: number (n) mixing ratios + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - } - for (int c = 0; c < mam_coupling::num_aero_species(); ++c) { - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, c); - std::cout< 0) { //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); @@ -69,10 +84,13 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) } } - //input for aci codes that existed in PBUF in EAM + //Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM //These outputs should come from the cloud macrophysics process (e.g., SHOC) - const auto m2 = m*m; - const auto s2 = s*s; + auto m2 = m*m; + m2.set_string("m^2"); + auto s2 = s*s; + s2.set_string("s^2"); + //FIXME BALLI: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time add_field("w_sec", scalar3d_layout_mid, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints @@ -82,7 +100,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("kvh", scalar3d_layout_mid, m2/s, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables - FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_modes_} }; // mid points + num_aero_modes_ = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes_} }; // mid points add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols @@ -96,11 +115,177 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) void MAMAci::initialize_impl(const RunType run_type) { m_atm_logger->log(ekat::logger::LogLevel::info,"ACI init = "); - std::cout<<"BALLI==============================="<log(ekat::logger::LogLevel::info,"ACI run = "); + + //---------------------------------------------------------- + // Convert from omega to w (vertical velocity) + // Negative omega means rising motion + //--------------------------------------------------------- + + //Get physical constants + using C = physics::Constants; + + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + + + /* + NOTE: All the inputs are available to compute w0 + Fortran code: + w0 = 0 + rho(:,:) = -999.0_r8 + do kk = top_lev, pver + do icol = 1, ncol + rho(icol,kk) = pmid(icol,kk)/(rair*temperature(icol,kk)) + w0(icol,kk) = -1._r8*omega(icol,kk)/(rho(icol,kk)*gravit) + enddo + enddo */ + + //--------------------------------------------------------- + //Compute TKE using "w_sec" + //--------------------------------------------------------- + + /* + NOTE: All the inputs are available to compute tke + Fortran code: + tke(:ncol,:) = (3._r8/2._r8)*w_sec(:ncol,:) + */ + //--------------------------------------------------------- + // Compute subgrid scale velocities(wsub, wsig and wsubice) + //--------------------------------------------------------- + + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + /* + NOTE: All the inputs are available to compute wsub, wsig and wsubice + "min_max_bound" is present in MAM4xx utils + + Fortran code: + wsub(:ncol,:top_lev-1) = wsubmin + wsubice(:ncol,:top_lev-1) = 0.001_r8 + wsig(:ncol,:top_lev-1) = 0.001_r8 + + do kk = top_lev, pver + do icol = 1, ncol + wsub(icol,kk) = sqrt(0.5_r8*(tke(icol,kk) + tke(icol,kk+1))*(2._r8/3._r8)) + wsig(icol,kk) = min_max_bound(0.001_r8, 10._r8, wsub(icol,kk)) + wsubice(icol,kk) = min_max_bound(0.2_r8, 10._r8, wsub(icol,kk)) + wsub(icol,kk) = max(wsubmin, wsub(icol,kk)) + end do + end do + */ + + + //--------------------------------------------------------- + // Compute subgrid mean updraft velocity (w2) + //--------------------------------------------------------- + + /* + NOTE: All inputs are available. "subgrid_mean_updraft" is not ported yet but it is a very small routine + Fortran code: + w2(1:ncol,1:pver) = 0._r8 + call subgrid_mean_updraft(ncol, w0, wsig, &!in + w2) !out + */ + + //------------------------------------------------------------- + // Get number of activated aerosol for ice nucleation (naai) + // from ice nucleation + //------------------------------------------------------------- + + /* + NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". + Only output we care about is "naai", "naai_hom" is never used anywhere + + Fortran code: + call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & ! input + rho, wsubice, strat_cld_frac, dgnum, & ! input + naai, naai_hom) ! output + */ + + //------------------------------------------------------------- + // Get old and new liquid cloud fractions when amount of cloud + // is above qsmall threshold value + //------------------------------------------------------------- + + static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) + + /* + MUST FIXME NOTE: We need old and new liquid cloud fractions here. + We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to + store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make + a note of it and use the new cloud fraction for the + old cloud fraction. + + Fortran code: + liqcldf(:ncol,:pver) = liq_strat_cld_frac(:ncol,:pver) + lcldn = 0._r8 + lcldo = 0._r8 + do kk = top_lev, pver + do icol = 1, ncol + qcld = qc(icol,kk) + qi(icol,kk) + if (qcld > qsmall) then + lcldn(icol,kk)=liqcldf(icol,kk) + lcldo(icol,kk)=liqcldfo(icol,kk) + end if + end do + end do + */ + + //------------------------------------------------------------- + // Save cloud borne aerosols to be used in the heterozenous + // freezing before they are changed by the droplet activation + // process. This is only a select subset of cloud borne + // aerosols, not all the cloud borne aerosols. + //------------------------------------------------------------- + + /*NOTE: We probably need to store indices for the select few + cloud borne aerosols + + Fortran code: + lchnk_zb = lchnk - begchunk + ! save copy of cloud borne aerosols for use in heterogeneous freezing before + !we change it in dropmixnuc + do ispec = 1, ncnst + call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) + aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) + aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * rho(:ncol,:) + enddo + */ + //------------------------------------------------------------- + // Compute activated fraction of aerosols + //------------------------------------------------------------- + + + /* + NOTE: "deltain" is the model time step. "state_q" is a combination of tracers + fields with "int_mmr_field_name" and "int_nmr_field_name". "z_mid" is computed. + "qqcw" is the combination of cld_mmr_field_name and cld_nmr_field_name. + The output "ptend" will have tendencies for interstitial and cloud borne aerosols. + + Fortan code: + call dropmixnuc(lchnk, ncol, deltatin, T_mid, p_mid, p_int, p_del, rpdel, z_mid, & ! in + state_q, nc, kvh, wsub, lcldn, lcldo, & ! in + qqcw, & ! inout + ptend, nctend_mixnuc, factnum) !out + */ + + + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + + /* + call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, & ! in + qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in + frzimm, frzcnt, frzdep) + */ + } void MAMAci::finalize_impl(){ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 232bcf95abc..21a8caeee2a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -1,9 +1,15 @@ #ifndef EAMXX_MAM_ACI_HPP #define EAMXX_MAM_ACI_HPP +//For MAM4 aerosol configuration #include + +//For declaring ACI class derived from atm process class #include +//For physical constants +#include "physics/share/physics_constants.hpp" + namespace scream { @@ -31,7 +37,7 @@ class MAMAci final : public scream::AtmosphereProcess { int ncol_, nlev_; // number of aerosol modes - int num_modes_; + int num_aero_modes_; // physics grid for column information diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index a14ec51d036..a7103ad01a8 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -27,10 +27,14 @@ initial_conditions: T_mid: 273.0 p_mid: 1.e5 num_a1: 1.e5 + num_c1: 1.e5 bc_a1: 1.e-5 num_a2: 1e5 num_a3: 1e5 num_a4: 1e5 + num_c2: 1e5 + num_c3: 1e5 + num_c4: 1e5 soa_a1: 1e-5 so4_a1: 1e-5 pom_a1: 1e-5 From 6ebbf3a70cb6a80ceb6d3ac4963754ea7ce97b89 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 28 Nov 2023 14:58:03 -0800 Subject: [PATCH 013/142] Cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 32 ++++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 32 +++++++++++++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7e35ef92313..7fccef52f05 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -14,7 +14,7 @@ MAMAci::MAMAci( //Return type of the process -AtmosphereProcessType MAMAci::type() const { +AtmosphereProcessType MAMAci::type() const{ return AtmosphereProcessType::Physics; } @@ -25,9 +25,9 @@ std::string MAMAci::name() const{ //set grid for all the inputs and outputs void MAMAci::set_grids(const std::shared_ptr grids_manager) { - m_atm_logger->log(ekat::logger::LogLevel::info,"ACI set grid = "); + m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI set grid"); - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); //Use physics grid const auto& grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -56,6 +56,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry + // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { //interstitial aerosol tracers of interest: number (n) mixing ratios @@ -91,7 +93,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) auto s2 = s*s; s2.set_string("s^2"); - //FIXME BALLI: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time + //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time add_field("w_sec", scalar3d_layout_mid, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints auto nondim = Units::nondimensional(); @@ -114,11 +116,25 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) } void MAMAci::initialize_impl(const RunType run_type) { - m_atm_logger->log(ekat::logger::LogLevel::info,"ACI init = "); + m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); + /* + NOTE: All other inputs should follow the way "pseudo_density" is initialized + */ + const auto& p_del = get_field_in("pseudo_density").get_view(); + + // set atmosphere state data + pdel_ = p_del; + + /* + NOTE: All derived variables (like rpdel and geopotential height) should be computed in + preprocess struct + */ + //preprocess_.set_variables(pdel_); + } void MAMAci::run_impl(const double dt) { - m_atm_logger->log(ekat::logger::LogLevel::info,"ACI run = "); + m_atm_logger->log(ekat::logger::LogLevel::info,"calling ACI run"); //---------------------------------------------------------- // Convert from omega to w (vertical velocity) @@ -259,7 +275,7 @@ void MAMAci::run_impl(const double dt) { // Compute activated fraction of aerosols //------------------------------------------------------------- - + std::cout<<"pdel_ in run_impl is:"<log(ekat::logger::LogLevel::info,"ACI final = "); + m_atm_logger->log(ekat::logger::LogLevel::info,"calling ACI final"); } } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 21a8caeee2a..7b63eb83fdd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -15,6 +15,13 @@ namespace scream class MAMAci final : public scream::AtmosphereProcess { + + using KT = ekat::KokkosTypes; + + // views for single- and multi-column data + using const_view_2d = typename KT::template view_2d; + + public: // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); @@ -38,6 +45,31 @@ class MAMAci final : public scream::AtmosphereProcess { // number of aerosol modes int num_aero_modes_; + + + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + + + //const_view_2d pdel_; // hydrostatic "pressure thickness" at grid + // interfaces [Pa] + + // assigns local variables + void set_variables(const const_view_2d& pdel) { + //p1del_ = pdel; + } // set_variables + }; // MAMAci::Preprocess + + + // pre- and postprocessing scratch pads + Preprocess preprocess_; + + // local atmospheric state column variables + const_view_2d pdel_; // hydrostatic "pressure thickness" at grid + // interfaces [Pa] // physics grid for column information From a05b9753a9cd86fd3fbd988865174b6e13b81960 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 30 Nov 2023 14:25:06 -0700 Subject: [PATCH 014/142] Add some scratch views to store density and velocity. --- .../mam/eamxx_mam_aci_process_interface.cpp | 21 ++++++++++++++++--- .../mam/eamxx_mam_aci_process_interface.hpp | 16 +++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7fccef52f05..83c3da5f2d4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -33,6 +33,9 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -120,10 +123,11 @@ void MAMAci::initialize_impl(const RunType run_type) { /* NOTE: All other inputs should follow the way "pseudo_density" is initialized */ - const auto& p_del = get_field_in("pseudo_density").get_view(); - // set atmosphere state data - pdel_ = p_del; + pdel_ = get_field_in("pseudo_density").get_view(); + omega_ = get_field_in("omega").get_view(); + p_mid_ = get_field_in("p_mid").get_view(); + T_mid_ = get_field_in("T_mid").get_view(); /* NOTE: All derived variables (like rpdel and geopotential height) should be computed in @@ -147,6 +151,17 @@ void MAMAci::run_impl(const double dt) { static constexpr auto gravit = C::gravit; // Gravity [m/s2] static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + // NOTE: All the inputs are available to compute w0 + for (int icol = 0; icol < ncol_; ++icol) { + for (int kk = 0; kk< top_lev_; ++kk) { + w0_(icol,kk) = 0; + rho_(icol, kk) = -999.0; + } + for (int kk = top_lev_; kk < nlev_; ++kk) { + rho_(icol,kk) = p_mid_(icol,kk)/(rair*T_mid_(icol,kk)); + w0_(icol,kk) = -1.0*omega_(icol,kk)/(rho_(icol,kk)*gravit); + } + } /* NOTE: All the inputs are available to compute w0 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 7b63eb83fdd..fe90d999195 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -20,7 +20,15 @@ class MAMAci final : public scream::AtmosphereProcess { // views for single- and multi-column data using const_view_2d = typename KT::template view_2d; - + using view_2d = typename KT::template view_2d; + // rho is air density [kg/m3] + view_2d rho_; + + // w0_ is large scale velocity (m/s) + view_2d w0_; + + // Top level for troposphere cloud physics + const int top_lev_ = 6; public: // Constructor @@ -70,8 +78,10 @@ class MAMAci final : public scream::AtmosphereProcess { // local atmospheric state column variables const_view_2d pdel_; // hydrostatic "pressure thickness" at grid // interfaces [Pa] - - + const_view_2d omega_; // Vertical pressure velocity [Pa/s] at midpoints + const_view_2d p_mid_; // Total pressure [Pa] at midpoints + const_view_2d T_mid_; // Temperature[K] at midpoints + // physics grid for column information std::shared_ptr grid_; }; // MAMAci From 82eb4d87446b95a6c613667f4bde5d9d84d59304 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 5 Dec 2023 09:05:25 -0700 Subject: [PATCH 015/142] Add subgrid_mean_updraft implementation. --- components/eamxx/CMakeLists.txt | 2 +- .../mam/eamxx_mam_aci_process_interface.cpp | 155 +++++++++++------- .../mam/eamxx_mam_aci_process_interface.hpp | 9 +- 3 files changed, 107 insertions(+), 59 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 8567512abf2..f0afd1c3045 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -546,7 +546,7 @@ if (SCREAM_DOUBLE_PRECISION) set(SCREAM_Fortran_FLAGS -real-size 64) elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") #find what is crayclang id in cmake - set(SCREAM_Fortran_FLAGS -s default32 -eZ) + set(SCREAM_Fortran_FLAGS -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 83c3da5f2d4..de5478873b6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,8 +1,57 @@ #include - namespace scream { +KOKKOS_INLINE_FUNCTION +Real subgrid_mean_updraft(const Real w0, const Real wsig) +{ +/* --------------------------------------------------------------------------------- + Purpose: Calculate the mean updraft velocity inside a GCM grid assuming the + vertical velocity distribution is Gaussian and peaks at the + GCM resolved large-scale vertical velocity. + When icenul_wsub_scheme = 2, the model uses the mean updraft velocity as the + characteristic updraft velocity to calculate the ice nucleation rate. + Author: Kai Zhang (kai.zhang@pnnl.gov) + Last Modified: Oct, 2015 +-------------------------------------------------------------------------------- */ + +// interface + +// in :: wsig standard deviation (m/s) +// in :: w0 large scale vertical velocity (m/s) +// out:: mean updraft velocity(m/s) -> characteristic w* + + // FIXME should nbin be a user parameter? + const int nbin = 50; + + using C = physics::Constants; + constexpr Real pi = C::Pi; + Real zz[nbin], wa = 0, ww = 0; + int kp = 0; + + const Real sigma = haero::max(0.001, wsig); + const Real wlarge = w0; + + const Real xx = 6.0 * sigma / nbin; + + for (int ibin = 0; ibin < nbin; ++ibin) { + Real yy = wlarge - 3.0*sigma + 0.5*xx; + yy += (ibin-1)*xx; + // wbar = integrator < w * f(w) * dw > + zz[ibin] = yy * haero::exp(-1.0*haero::square(yy-wlarge)/(2*sigma*sigma))/(sigma*haero::sqrt(2*pi))*xx; + } + for (int ibin = 0; ibin < nbin; ++ibin) { + if (zz[ibin] > 0) ++kp, wa += zz[ibin]; + } + if (kp) { + // wbar = integrator < w * f(w) * dw > + ww = wa; + } else { + ww = 0.001; + } + return ww; +} + //FIXME: The following variables are namelist variables Real wsubmin = 1; @@ -35,7 +84,12 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); - + Kokkos::resize(tke_, ncol_, nlev_+1); + Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(wsubice_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -128,6 +182,7 @@ void MAMAci::initialize_impl(const RunType run_type) { omega_ = get_field_in("omega").get_view(); p_mid_ = get_field_in("p_mid").get_view(); T_mid_ = get_field_in("T_mid").get_view(); + w_sec_ = get_field_in("w_sec").get_view(); /* NOTE: All derived variables (like rpdel and geopotential height) should be computed in @@ -152,75 +207,61 @@ void MAMAci::run_impl(const double dt) { static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] // NOTE: All the inputs are available to compute w0 - for (int icol = 0; icol < ncol_; ++icol) { - for (int kk = 0; kk< top_lev_; ++kk) { + auto team_policy = haero::ThreadTeamPolicy(ncol_, Kokkos::AUTO); + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev_), KOKKOS_LAMBDA(int kk) { w0_(icol,kk) = 0; rho_(icol, kk) = -999.0; - } - for (int kk = top_lev_; kk < nlev_; ++kk) { + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev_, nlev_), KOKKOS_LAMBDA(int kk) { rho_(icol,kk) = p_mid_(icol,kk)/(rair*T_mid_(icol,kk)); w0_(icol,kk) = -1.0*omega_(icol,kk)/(rho_(icol,kk)*gravit); - } - } - - /* - NOTE: All the inputs are available to compute w0 - Fortran code: - w0 = 0 - rho(:,:) = -999.0_r8 - do kk = top_lev, pver - do icol = 1, ncol - rho(icol,kk) = pmid(icol,kk)/(rair*temperature(icol,kk)) - w0(icol,kk) = -1._r8*omega(icol,kk)/(rho(icol,kk)*gravit) - enddo - enddo */ + }); + }); - //--------------------------------------------------------- - //Compute TKE using "w_sec" - //--------------------------------------------------------- - - /* - NOTE: All the inputs are available to compute tke - Fortran code: - tke(:ncol,:) = (3._r8/2._r8)*w_sec(:ncol,:) - */ + //--------------------------------------------------------- + //Compute TKE using "w_sec" + //--------------------------------------------------------- + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // FIXME Is this the correct boundary condition for tke at the surface? + tke_(icol,nlev_) = 0; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev_), KOKKOS_LAMBDA(int kk) { + tke_(icol,kk) = (3.0/2.0)*w_sec_(icol,kk); + }); + }); //--------------------------------------------------------- // Compute subgrid scale velocities(wsub, wsig and wsubice) //--------------------------------------------------------- - // More refined computation of sub-grid vertical velocity - // Set to be zero at the surface by initialization. - /* - NOTE: All the inputs are available to compute wsub, wsig and wsubice - "min_max_bound" is present in MAM4xx utils - - Fortran code: - wsub(:ncol,:top_lev-1) = wsubmin - wsubice(:ncol,:top_lev-1) = 0.001_r8 - wsig(:ncol,:top_lev-1) = 0.001_r8 - - do kk = top_lev, pver - do icol = 1, ncol - wsub(icol,kk) = sqrt(0.5_r8*(tke(icol,kk) + tke(icol,kk+1))*(2._r8/3._r8)) - wsig(icol,kk) = min_max_bound(0.001_r8, 10._r8, wsub(icol,kk)) - wsubice(icol,kk) = min_max_bound(0.2_r8, 10._r8, wsub(icol,kk)) - wsub(icol,kk) = max(wsubmin, wsub(icol,kk)) - end do - end do - */ - + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev_), KOKKOS_LAMBDA(int kk) { + wsub_(icol,kk) = wsubmin; + wsubice_(icol,kk) = 0.001; + wsig_(icol,kk) = 0.001; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev_, nlev_), KOKKOS_LAMBDA(int kk) { + wsub_(icol,kk) = haero::sqrt(0.5*(tke_(icol,kk) + tke_(icol,kk+1))*(2.0/3.0)); + wsig_(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub_(icol,kk)); + wsubice_(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub_(icol,kk)); + wsub_(icol,kk) = haero::max(wsubmin, wsub_(icol,kk)); + }); + }); //--------------------------------------------------------- // Compute subgrid mean updraft velocity (w2) //--------------------------------------------------------- - /* - NOTE: All inputs are available. "subgrid_mean_updraft" is not ported yet but it is a very small routine - Fortran code: - w2(1:ncol,1:pver) = 0._r8 - call subgrid_mean_updraft(ncol, w0, wsig, &!in - w2) !out - */ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev_), KOKKOS_LAMBDA(int kk) { + w2_(icol,kk) = subgrid_mean_updraft(w0_(icol,kk), wsig_(icol,kk)); + }); + }); //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index fe90d999195..8c0decf0925 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -27,7 +27,13 @@ class MAMAci final : public scream::AtmosphereProcess { // w0_ is large scale velocity (m/s) view_2d w0_; + // turbulent kinetic energy [m^2/s^2] + view_2d tke_; + + // Subgrid scale velocities + view_2d wsub_, wsubice_, wsig_, w2_; // Top level for troposphere cloud physics + // FIXME: This should be read in to make user selectable. const int top_lev_ = 6; public: @@ -81,7 +87,8 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d omega_; // Vertical pressure velocity [Pa/s] at midpoints const_view_2d p_mid_; // Total pressure [Pa] at midpoints const_view_2d T_mid_; // Temperature[K] at midpoints - + const_view_2d w_sec_; // Vertical velocity variance + // physics grid for column information std::shared_ptr grid_; }; // MAMAci From 512818d389e55014eb6225a4e3fbb6c8bc8fd421 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 7 Dec 2023 08:32:47 -0700 Subject: [PATCH 016/142] Add call to nucleate_ice_ This required a number input fields many of which are not in the regression test input file. Will need to find a way to add them. Next is the dropmixnuc function call which is also has quite a few input fields. --- .../mam/eamxx_mam_aci_process_interface.cpp | 239 +++++++++++++++--- .../mam/eamxx_mam_aci_process_interface.hpp | 34 +++ 2 files changed, 234 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index de5478873b6..7a4368d327f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,4 +1,6 @@ #include +#include "ekat/util/ekat_units.hpp" +#include "mam4xx/aero_config.hpp" namespace scream { @@ -52,8 +54,8 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) return ww; } - //FIXME: The following variables are namelist variables - Real wsubmin = 1; +//FIXME: The following variables are namelist variables +const Real wsubmin = 1; MAMAci::MAMAci( const ekat::Comm& comm, @@ -89,6 +91,9 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) Kokkos::resize(wsubice_, ncol_, nlev_); Kokkos::resize(wsig_, ncol_, nlev_); Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(lcldn_, ncol_, nlev_); + Kokkos::resize(lcldo_, ncol_, nlev_); + Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -102,7 +107,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) q_unit.set_string("kg/kg"); auto n_unit = 1/kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); - + auto nondim = Units::nondimensional(); // atmospheric quantities add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] @@ -111,7 +116,26 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces + add_field("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] + //add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] + //add_field("q_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_so4", scalar3d_layout_mid, q_unit, grid_name); // Sulfuric Acid mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] + //add_field("n_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] + //add_field("n_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] + + add_field("nihf",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] + add_field("niim",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] + add_field("nidep",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] + add_field("nimey",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + add_field("naai_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] + add_field("naai",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry @@ -125,7 +149,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios @@ -151,9 +175,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) s2.set_string("s^2"); //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time - add_field("w_sec", scalar3d_layout_mid, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints + add_field("w_sec", scalar3d_layout_int, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints - auto nondim = Units::nondimensional(); add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints add_field("kvh", scalar3d_layout_mid, m2/s, grid_name); // Eddy diffusivity for heat @@ -161,7 +184,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) // Layout for 4D (2d horiz X 1d vertical x number of modes) variables num_aero_modes_ = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes_} }; // mid points - add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols /*NOTE on other inputs for the aci process: @@ -183,6 +206,34 @@ void MAMAci::initialize_impl(const RunType run_type) { p_mid_ = get_field_in("p_mid").get_view(); T_mid_ = get_field_in("T_mid").get_view(); w_sec_ = get_field_in("w_sec").get_view(); + qv_dry_ = get_field_in("qv").get_view(); + cldfrac_ = get_field_in("cldfrac_tot").get_view(); + Kokkos::resize(w_updraft_, ncol_, nlev_); // w_updraft_ = get_field_in("w_updraft").get_view(); + Kokkos::resize(q_coarse_dst_, ncol_, nlev_); // q_coarse_dst_ = get_field_in("q_coarse_dst").get_view(); + Kokkos::resize(q_coarse_nacl_, ncol_, nlev_); // q_coarse_nacl_ = get_field_in("q_coarse_nacl").get_view(); + Kokkos::resize(q_coarse_so4_, ncol_, nlev_); // q_coarse_so4_ = get_field_in("q_coarse_so4").get_view(); + Kokkos::resize(q_coarse_mom_, ncol_, nlev_); // q_coarse_mom_ = get_field_in("q_coarse_mom").get_view(); + Kokkos::resize(q_coarse_bc_, ncol_, nlev_); // q_coarse_bc_ = get_field_in("q_coarse_bc").get_view(); + Kokkos::resize(q_coarse_pom_, ncol_, nlev_); // q_coarse_pom_ = get_field_in("q_coarse_pom").get_view(); + Kokkos::resize(q_coarse_soa_, ncol_, nlev_); // q_coarse_soa_ = get_field_in("q_coarse_soa").get_view(); + Kokkos::resize(n_coarse_, ncol_, nlev_); // n_coarse_ = get_field_in("n_coarse").get_view(); + Kokkos::resize(n_aitken_, ncol_, nlev_); // n_aitken_ = get_field_in("n_aitken").get_view(); + dgnum_ = get_field_out("dgnum").get_view(); + nihf_ = get_field_out("nihf").get_view(); + niim_ = get_field_out("niim").get_view(); + nidep_ = get_field_out("nidep").get_view(); + nimey_ = get_field_out("nimey").get_view(); + naai_hom_ = get_field_out("naai_hom").get_view(); + naai_ = get_field_out("naai").get_view(); + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + qc_ = get_field_in("qc").get_view(); + qi_ = get_field_in("qi").get_view(); + + + // configure the nucleation parameterization + mam4::NucleateIce::Config config; + mam4::AeroConfig aero_config; + nucleate_ice_.init(aero_config, config); /* NOTE: All derived variables (like rpdel and geopotential height) should be computed in @@ -206,17 +257,57 @@ void MAMAci::run_impl(const double dt) { static constexpr auto gravit = C::gravit; // Gravity [m/s2] static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + // Alias member variables + auto w0 = w0_; + auto w2 = w2_; + auto rho = rho_; + auto tke = tke_; + auto T_mid = T_mid_; + auto p_mid = p_mid_; + auto w_sec = w_sec_; + auto wsub = wsub_; + auto wsubice = wsubice_; + auto wsig = wsig_; + auto qv_dry = qv_dry_; + auto cldfrac = cldfrac_; + auto w_updraft = w_updraft_; + const auto nlev = nlev_; + const auto top_lev = top_lev_; + + auto q_coarse_dst = q_coarse_dst_; + auto q_coarse_nacl = q_coarse_nacl_; + auto q_coarse_so4 = q_coarse_so4_; + auto q_coarse_mom = q_coarse_mom_; + auto q_coarse_bc = q_coarse_bc_; + auto q_coarse_pom = q_coarse_pom_; + auto q_coarse_soa = q_coarse_soa_; + auto n_coarse = n_coarse_; + auto n_aitken = n_aitken_; + auto dgnum = dgnum_; + auto nihf = nihf_; + auto niim = niim_; + auto nidep = nidep_; + auto nimey = nimey_; + auto naai_hom = naai_hom_; + auto naai = naai_; + auto liqcldf = liqcldf_; + auto qc = qc_; + auto qi = qi_; + auto lcldn = lcldn_; + auto lcldo = lcldo_; + auto aitken_dry_dia = aitken_dry_dia_; + // NOTE: All the inputs are available to compute w0 auto team_policy = haero::ThreadTeamPolicy(ncol_, Kokkos::AUTO); Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev_), KOKKOS_LAMBDA(int kk) { - w0_(icol,kk) = 0; - rho_(icol, kk) = -999.0; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol,kk) = 0; + rho(icol, kk) = -999.0; }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev_, nlev_), KOKKOS_LAMBDA(int kk) { - rho_(icol,kk) = p_mid_(icol,kk)/(rair*T_mid_(icol,kk)); - w0_(icol,kk) = -1.0*omega_(icol,kk)/(rho_(icol,kk)*gravit); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol,kk) = p_mid_(icol,kk)/(rair*T_mid_(icol,kk)); + w0(icol,kk) = -1.0*omega_(icol,kk)/(rho_(icol,kk)*gravit); }); }); @@ -226,9 +317,10 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // FIXME Is this the correct boundary condition for tke at the surface? - tke_(icol,nlev_) = 0; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev_), KOKKOS_LAMBDA(int kk) { - tke_(icol,kk) = (3.0/2.0)*w_sec_(icol,kk); + // TKE seems to be at interfaces but w_sec is at cell centers so this + // descrepensy needs to be worked out. + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { + tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); }); }); //--------------------------------------------------------- @@ -239,16 +331,16 @@ void MAMAci::run_impl(const double dt) { // Set to be zero at the surface by initialization. Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev_), KOKKOS_LAMBDA(int kk) { - wsub_(icol,kk) = wsubmin; - wsubice_(icol,kk) = 0.001; - wsig_(icol,kk) = 0.001; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = wsubmin; + wsubice(icol,kk) = 0.001; + wsig(icol,kk) = 0.001; }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev_, nlev_), KOKKOS_LAMBDA(int kk) { - wsub_(icol,kk) = haero::sqrt(0.5*(tke_(icol,kk) + tke_(icol,kk+1))*(2.0/3.0)); - wsig_(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub_(icol,kk)); - wsubice_(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub_(icol,kk)); - wsub_(icol,kk) = haero::max(wsubmin, wsub_(icol,kk)); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = haero::sqrt(0.5*(tke_(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); + wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); + wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); + wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); }); }); @@ -258,26 +350,84 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev_), KOKKOS_LAMBDA(int kk) { - w2_(icol,kk) = subgrid_mean_updraft(w0_(icol,kk), wsig_(icol,kk)); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); }); }); - //------------------------------------------------------------- - // Get number of activated aerosol for ice nucleation (naai) - // from ice nucleation - //------------------------------------------------------------- + //------------------------------------------------------------- + // Get number of activated aerosol for ice nucleation (naai) + // from ice nucleation + //------------------------------------------------------------- + using view_1d = typename KokkosTypes::template view_1d; + view_1d dummy("DummyView", nlev); + + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); + }); + }); + + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + // Set up an atmosphere, surface, diagnostics, pronostics and tendencies class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, + ekat::subview(T_mid, icol), + ekat::subview(p_mid, icol), + ekat::subview(qv_dry, icol), + dummy, dummy, dummy, dummy, dummy, dummy, + ekat::subview(cldfrac, icol), + ekat::subview(w_updraft, icol), pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs(nlev); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(q_coarse_dst, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(q_coarse_nacl, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(q_coarse_so4, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(q_coarse_mom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(q_coarse_bc, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(q_coarse_pom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(q_coarse_soa, icol); + progs.n_mode_i[coarse_idx] = ekat::subview(n_coarse, icol); + progs.n_mode_i[aitken_idx] = ekat::subview(n_aitken, icol); + + // nucleation doesn't use any diagnostics, so it's okay to leave this alone + // for now + mam4::Diagnostics diags(nlev); + diags.dry_geometric_mean_diameter_i[aitken_idx] = ekat::subview(aitken_dry_dia, icol); + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t=0, dt=0; + nucleate_ice_.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); /* - NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". - Only output we care about is "naai", "naai_hom" is never used anywhere + NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". + Only output we care about is "naai", "naai_hom" is never used anywhere - Fortran code: - call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & ! input - rho, wsubice, strat_cld_frac, dgnum, & ! input - naai, naai_hom) ! output + Fortran code: + call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & ! input + rho, wsubice, strat_cld_frac, dgnum, & ! input + naai, naai_hom) ! output */ - + }); //------------------------------------------------------------- // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value @@ -291,7 +441,18 @@ void MAMAci::run_impl(const double dt) { store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make a note of it and use the new cloud fraction for the old cloud fraction. - + */ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + const Real qcld = qc(icol,kk) + qi(icol,kk); + if (qcld > qsmall) { + lcldn(icol,kk)=liqcldf(icol,kk); + lcldo(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old + } + }); + }); + /* Fortran code: liqcldf(:ncol,:pver) = liq_strat_cld_frac(:ncol,:pver) lcldn = 0._r8 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 8c0decf0925..b959990d9cd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -18,9 +18,15 @@ class MAMAci final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; + mam4::NucleateIce nucleate_ice_; + // views for single- and multi-column data + using view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; using view_2d = typename KT::template view_2d; + using view_3d = typename KT::template view_3d; + using const_view_3d = typename KT::template view_3d; + // rho is air density [kg/m3] view_2d rho_; @@ -30,6 +36,34 @@ class MAMAci final : public scream::AtmosphereProcess { // turbulent kinetic energy [m^2/s^2] view_2d tke_; + const_view_2d qv_dry_; + const_view_2d cldfrac_; + //const_view_2d w_updraft_; + view_2d w_updraft_; + + view_2d aitken_dry_dia_; + view_2d q_coarse_dst_; + view_2d q_coarse_nacl_; + view_2d q_coarse_so4_; + view_2d q_coarse_mom_; + view_2d q_coarse_bc_; + view_2d q_coarse_pom_; + view_2d q_coarse_soa_; + view_2d n_coarse_; + view_2d n_aitken_; + view_3d dgnum_; + view_2d nihf_; + view_2d niim_; + view_2d nidep_; + view_2d nimey_; + view_2d naai_hom_; + view_2d naai_; + const_view_2d liqcldf_; + const_view_2d qc_; + const_view_2d qi_; + view_2d lcldn_; + view_2d lcldo_; + // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; // Top level for troposphere cloud physics From c1a663507929c2c6b93b2490e737943b7ed396ce Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 19 Dec 2023 15:23:26 -0700 Subject: [PATCH 017/142] Update eamxx_mam_aci_process_interface to call dropmixnuc. Since dropmixnuc takes about 75 input arguments, this is no trivial task. Now that the inputs are defined and allocated, need to clean this up with a few helper classes like Jeff Johnson has used for other cases. --- .../mam/eamxx_mam_aci_process_interface.cpp | 278 +++++++++++++++++- .../mam/eamxx_mam_aci_process_interface.hpp | 44 ++- 2 files changed, 306 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7a4368d327f..9e70dc8a477 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,6 +1,7 @@ #include #include "ekat/util/ekat_units.hpp" #include "mam4xx/aero_config.hpp" +#include "mam4xx/ndrop.hpp" namespace scream { @@ -25,6 +26,7 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) // FIXME should nbin be a user parameter? const int nbin = 50; + using C = physics::Constants; constexpr Real pi = C::Pi; @@ -84,17 +86,21 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - Kokkos::resize(rho_, ncol_, nlev_); - Kokkos::resize(w0_, ncol_, nlev_); - Kokkos::resize(tke_, ncol_, nlev_+1); - Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + Kokkos::resize(tke_, ncol_, nlev_+1); + Kokkos::resize(wsub_, ncol_, nlev_); Kokkos::resize(wsubice_, ncol_, nlev_); - Kokkos::resize(wsig_, ncol_, nlev_); - Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(lcldn_, ncol_, nlev_); - Kokkos::resize(lcldo_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(lcldn_, ncol_, nlev_); + Kokkos::resize(lcldo_, ncol_, nlev_); Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); + Kokkos::resize(rpdel_, ncol_, nlev_); + for (int i=0; i<15; ++i) { + Kokkos::resize(scratch_mem_[i],ncol_, nlev_); + } // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -118,7 +124,9 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces add_field("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] + + + // FIXME: These should all come from the intput file. Might need to be added to the input file? //add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] //add_field("q_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] //add_field("q_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] @@ -129,6 +137,12 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) //add_field("q_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] //add_field("n_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] //add_field("n_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] + //add_field("zm", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) + //add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] + //add_field("ncldwtr", , n_unit, grid_name); // initial droplet number mixing ratio [#/kg] + //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] + //add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] + add_field("nihf",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] add_field("niim",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] @@ -136,6 +150,22 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("nimey",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] add_field("naai_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] add_field("naai",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] + add_field("qcld",scalar3d_layout_mid , n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] + add_field("ptend_q", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne aerosols [#/kg] + add_field("tendnd",scalar3d_layout_mid , n_unit/s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] + add_field("factnum", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::AeroConfig::num_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] + add_field("ndropcol",scalar3d_layout_mid , n_unit/s, grid_name); // + add_field("ndropmix",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio tendency due to mixing [#/kg/s] + add_field("nsource",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] + add_field("wtke",scalar3d_layout_mid , n_unit/s, grid_name); // + add_field("ccn", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, grid_name); //number conc of aerosols activated at supersat [#/m^3] + // note: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud [#/m^2/s]] + // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] + add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output + add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency + + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry @@ -179,7 +209,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints - add_field("kvh", scalar3d_layout_mid, m2/s, grid_name); // Eddy diffusivity for heat + add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables num_aero_modes_ = mam_coupling::num_aero_modes(); @@ -204,10 +234,11 @@ void MAMAci::initialize_impl(const RunType run_type) { pdel_ = get_field_in("pseudo_density").get_view(); omega_ = get_field_in("omega").get_view(); p_mid_ = get_field_in("p_mid").get_view(); + p_int_ = get_field_in("p_int").get_view(); T_mid_ = get_field_in("T_mid").get_view(); w_sec_ = get_field_in("w_sec").get_view(); qv_dry_ = get_field_in("qv").get_view(); - cldfrac_ = get_field_in("cldfrac_tot").get_view(); + // FIXME: These should all come from the intput file. Might need to be added to the input file? Kokkos::resize(w_updraft_, ncol_, nlev_); // w_updraft_ = get_field_in("w_updraft").get_view(); Kokkos::resize(q_coarse_dst_, ncol_, nlev_); // q_coarse_dst_ = get_field_in("q_coarse_dst").get_view(); Kokkos::resize(q_coarse_nacl_, ncol_, nlev_); // q_coarse_nacl_ = get_field_in("q_coarse_nacl").get_view(); @@ -218,6 +249,31 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(q_coarse_soa_, ncol_, nlev_); // q_coarse_soa_ = get_field_in("q_coarse_soa").get_view(); Kokkos::resize(n_coarse_, ncol_, nlev_); // n_coarse_ = get_field_in("n_coarse").get_view(); Kokkos::resize(n_aitken_, ncol_, nlev_); // n_aitken_ = get_field_in("n_aitken").get_view(); + Kokkos::resize(zm_, ncol_, nlev_); // zm_ = get_field_in("zm").get_view(); + Kokkos::resize(state_q_, ncol_, nlev_, mam4::ndrop::nvars); // state_q_ = get_field_in("state_q").get_view(); + Kokkos::resize(ncldwtr_, ncol_, nlev_); // ncldwtr_ = get_field_in("ncldwtr").get_view(); + Kokkos::resize(cldo_, ncol_, nlev_); // cldo_ = get_field_in("cldo").get_view(); + Kokkos::resize(qqcw_inp_, ncol_, nlev_, mam4::ndrop::ncnst_tot); // qqcw_inp_ = get_field_in("qqcw").get_view(); + + for (int i=0; i(); dgnum_ = get_field_out("dgnum").get_view(); nihf_ = get_field_out("nihf").get_view(); niim_ = get_field_out("niim").get_view(); @@ -225,9 +281,24 @@ void MAMAci::initialize_impl(const RunType run_type) { nimey_ = get_field_out("nimey").get_view(); naai_hom_ = get_field_out("naai_hom").get_view(); naai_ = get_field_out("naai").get_view(); + qcld_ = get_field_out("qcld").get_view(); + ptend_q_inp_ = get_field_out("ptend_q").get_view(); + tendnd_ = get_field_out("tendnd").get_view(); + factnum_ = get_field_out("factnum").get_view(); + ndropcol_ = get_field_out("ndropcol").get_view(); + ndropmix_ = get_field_out("ndropmix").get_view(); + nsource_ = get_field_out("nsource").get_view(); + wtke_ = get_field_out("wtke").get_view(); + ccn_ = get_field_out("ccn").get_view(); + coltend_outp_ = get_field_out("coltend").get_view(); + coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); + + + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); qc_ = get_field_in("qc").get_view(); qi_ = get_field_in("qi").get_view(); + kvh_ = get_field_in("kvh").get_view(); // configure the nucleation parameterization @@ -264,6 +335,8 @@ void MAMAci::run_impl(const double dt) { auto tke = tke_; auto T_mid = T_mid_; auto p_mid = p_mid_; + auto p_int = p_int_; + auto pdel = pdel_; auto w_sec = w_sec_; auto wsub = wsub_; auto wsubice = wsubice_; @@ -296,9 +369,67 @@ void MAMAci::run_impl(const double dt) { auto lcldn = lcldn_; auto lcldo = lcldo_; auto aitken_dry_dia = aitken_dry_dia_; + auto rpdel = rpdel_; + auto zm = zm_; + auto state_q = state_q_; + auto ncldwtr = ncldwtr_; + auto kvh = kvh_; + auto qcld = qcld_; + auto ptend_q_inp = ptend_q_inp_; + auto tendnd = tendnd_; + auto factnum = factnum_; + auto ndropcol = ndropcol_ ; + auto ndropmix = ndropmix_ ; + auto nsource = nsource_ ; + auto wtke = wtke_ ; + auto cldo = cldo_; + auto qqcw_inp = qqcw_inp_; + auto ccn = ccn_; + auto coltend_outp = coltend_outp_; + auto coltend_cw_outp = coltend_cw_outp_; + auto nact = nact_; + auto mact = mact_; + auto eddy_diff = scratch_mem_[0]; + auto zn = scratch_mem_[1]; + auto csbot = scratch_mem_[2]; + auto zs = scratch_mem_[3]; + auto overlapp = scratch_mem_[4]; + auto overlapm = scratch_mem_[5]; + auto eddy_diff_kp = scratch_mem_[6]; + auto eddy_diff_km = scratch_mem_[7]; + auto qncld = scratch_mem_[8]; + auto srcn = scratch_mem_[9]; + auto source = scratch_mem_[10]; + auto dz = scratch_mem_[11]; + auto csbot_cscen = scratch_mem_[12]; + auto raertend = scratch_mem_[13]; + auto qqcwtend = scratch_mem_[14]; - // NOTE: All the inputs are available to compute w0 auto team_policy = haero::ThreadTeamPolicy(ncol_, Kokkos::AUTO); + + // NOTE: All the inputs are available to compute w0 + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + for (int i=0; i; using const_view_3d = typename KT::template view_3d; + template + using view_4d = KT::view; + + // FIXME the time step for microphysics [s] need to get from the input + const Real dtmicro_ = .0001; + // rho is air density [kg/m3] view_2d rho_; @@ -37,7 +43,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d tke_; const_view_2d qv_dry_; - const_view_2d cldfrac_; + view_2d cldfrac_; //const_view_2d w_updraft_; view_2d w_updraft_; @@ -60,9 +66,38 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d naai_; const_view_2d liqcldf_; const_view_2d qc_; - const_view_2d qi_; + const_view_2d qi_; + const_view_2d kvh_; + view_2d cldo_; + + view_2d zm_; + view_3d state_q_; + view_2d ncldwtr_; + view_2d lcldn_; view_2d lcldo_; + view_2d qcld_; + view_2d tendnd_; + view_2d ptend_q_[mam4::ndrop::nvar_ptend_q]; + view_3d ptend_q_inp_; + view_3d factnum_; + view_3d qqcw_inp_; + view_2d qqcw_[mam4::ndrop::ncnst_tot]; + view_2d ndropcol_; + view_2d ndropmix_; + view_2d nsource_; + view_2d wtke_; + view_3d ccn_; + view_3d coltend_outp_; + view_2d coltend_[mam4::ndrop::ncnst_tot]; + view_3d coltend_cw_outp_; + view_2d coltend_cw_[mam4::ndrop::ncnst_tot]; + view_2d raercol_cw_[mam4::ndrop::pver][2]; + view_2d raercol_[mam4::ndrop::pver][2]; + view_3d nact_; + view_3d mact_; + view_2d scratch_mem_[15]; + // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; @@ -116,11 +151,12 @@ class MAMAci final : public scream::AtmosphereProcess { Preprocess preprocess_; // local atmospheric state column variables - const_view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] const_view_2d omega_; // Vertical pressure velocity [Pa/s] at midpoints const_view_2d p_mid_; // Total pressure [Pa] at midpoints + const_view_2d p_int_; // Total pressure [Pa] at interfaces const_view_2d T_mid_; // Temperature[K] at midpoints + const_view_2d pdel_; // pressure thickess of layer [Pa] + view_2d rpdel_; // Inverse of pdel_ const_view_2d w_sec_; // Vertical velocity variance // physics grid for column information From 7a5aaecd0ec27f897c94a59e1806cd63dcba5ebd Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 23 Jan 2024 13:21:48 -0700 Subject: [PATCH 018/142] Just stashing a working version. --- .../mam/eamxx_mam_aci_process_interface.cpp | 602 +++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 71 ++- 2 files changed, 489 insertions(+), 184 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 9e70dc8a477..1fb52659ac5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -2,9 +2,99 @@ #include "ekat/util/ekat_units.hpp" #include "mam4xx/aero_config.hpp" #include "mam4xx/ndrop.hpp" + namespace scream { +namespace +{ +void copy_scream_array_to_mam4xx( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d mam4xx_view[], + MAMAci::const_view_3d scream_view, + const int nlev, + const int num_views) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + for (int i=0; i; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol,kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol,kk) = p_mid(icol,kk)/(rair*T_mid(icol,kk)); + w0(icol,kk) = -1.0*omega(icol,kk)/(rho(icol,kk)*gravit); + }); + }); +} +void compute_tke_using_w_sec( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d tke, + MAMAci::const_view_2d w_sec, + const int nlev) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // FIXME Is this the correct boundary condition for tke at the surface? + // TKE seems to be at interfaces but w_sec is at cell centers so this + // descrepensy needs to be worked out. + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { + tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); + }); + }); +} +void compute_subgrid_scale_velocities( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d wsub, + MAMAci::view_2d wsubice, + MAMAci::view_2d wsig, + MAMAci::const_view_2d tke, + const Real wsubmin, + const int top_lev, + const int nlev) +{ + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = wsubmin; + wsubice(icol,kk) = 0.001; + wsig(icol,kk) = 0.001; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = haero::sqrt(0.5*(tke(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); + wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); + wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); + wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); + }); + }); +} + + KOKKOS_INLINE_FUNCTION Real subgrid_mean_updraft(const Real w0, const Real wsig) { @@ -55,6 +145,43 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) } return ww; } +void compute_subgrid_mean_updraft_velocities( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d w2, + MAMAci::const_view_2d w0, + MAMAci::const_view_2d wsig, + const int nlev) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); + }); + }); +} +void compute_aitken_dry_diameter( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d aitken_dry_dia, + MAMAci::const_view_3d dgnum, + const int top_lev) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); + }); + }); +} + +void compute_nucleate_ice_tendencies( + haero::ThreadTeamPolicy team_policy, + const int nlev) +{ +} + +} + //FIXME: The following variables are namelist variables const Real wsubmin = 1; @@ -99,7 +226,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) Kokkos::resize(rpdel_, ncol_, nlev_); for (int i=0; i<15; ++i) { - Kokkos::resize(scratch_mem_[i],ncol_, nlev_); + Kokkos::resize(dropmixnuc_scratch_mem_[i],ncol_, nlev_); } // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -117,31 +244,48 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) // atmospheric quantities add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid mass mixing ratio [kg/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces add_field("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + // + // + add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints // FIXME: These should all come from the intput file. Might need to be added to the input file? - //add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] - //add_field("q_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_so4", scalar3d_layout_mid, q_unit, grid_name); // Sulfuric Acid mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] - //add_field("n_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] - //add_field("n_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] + add_field("qi_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_so4", scalar3d_layout_mid, q_unit, grid_name); // Sulfuric Acid mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] + //add_field("qi_accum_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for accum mode [kg/kg] + //add_field("qi_accum_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for accum mode [kg/kg] + //add_field("qi_accum_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for accum mode [kg/kg] + //add_field("qi_accumn_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for primary carbon mode [kg/kg] + //add_field("qi_accumn_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for primary carbon mode [kg/kg]] + //add_field("qi_accumn_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for primary carbon mode [kg/kg]kg] + //add_field("qi_pcarbon_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for primary carbon mode [kg/kg] + //add_field("qi_pcarbon_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for primary carbon mode [kg/kg]] + //add_field("qi_pcarbon_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for primary carbon mode [kg/kg]kg] + //add_field("ni_accum", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] + //add_field("ni_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] + //add_field("ni_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] //add_field("zm", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) //add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] //add_field("ncldwtr", , n_unit, grid_name); // initial droplet number mixing ratio [#/kg] //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] - //add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] + // + add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] + add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] add_field("nihf",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] @@ -165,7 +309,6 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry @@ -214,7 +357,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) // Layout for 4D (2d horiz X 1d vertical x number of modes) variables num_aero_modes_ = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes_} }; // mid points - add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols /*NOTE on other inputs for the aci process: @@ -238,22 +381,53 @@ void MAMAci::initialize_impl(const RunType run_type) { T_mid_ = get_field_in("T_mid").get_view(); w_sec_ = get_field_in("w_sec").get_view(); qv_dry_ = get_field_in("qv").get_view(); + w_updraft_ = get_field_in("w_updraft").get_view(); // updraft velocity [m/s] + // + stratiform_cloud_fraction_ = get_field_out("stratiform_cloud_fraction").get_view(); + activation_fraction_accum_idx_ = get_field_out("activation_fraction_accum").get_view(); + activation_fraction_coarse_idx_ = get_field_out("activation_fraction_coarse").get_view(); + // FIXME: These should all come from the intput file. Might need to be added to the input file? - Kokkos::resize(w_updraft_, ncol_, nlev_); // w_updraft_ = get_field_in("w_updraft").get_view(); - Kokkos::resize(q_coarse_dst_, ncol_, nlev_); // q_coarse_dst_ = get_field_in("q_coarse_dst").get_view(); - Kokkos::resize(q_coarse_nacl_, ncol_, nlev_); // q_coarse_nacl_ = get_field_in("q_coarse_nacl").get_view(); - Kokkos::resize(q_coarse_so4_, ncol_, nlev_); // q_coarse_so4_ = get_field_in("q_coarse_so4").get_view(); - Kokkos::resize(q_coarse_mom_, ncol_, nlev_); // q_coarse_mom_ = get_field_in("q_coarse_mom").get_view(); - Kokkos::resize(q_coarse_bc_, ncol_, nlev_); // q_coarse_bc_ = get_field_in("q_coarse_bc").get_view(); - Kokkos::resize(q_coarse_pom_, ncol_, nlev_); // q_coarse_pom_ = get_field_in("q_coarse_pom").get_view(); - Kokkos::resize(q_coarse_soa_, ncol_, nlev_); // q_coarse_soa_ = get_field_in("q_coarse_soa").get_view(); - Kokkos::resize(n_coarse_, ncol_, nlev_); // n_coarse_ = get_field_in("n_coarse").get_view(); - Kokkos::resize(n_aitken_, ncol_, nlev_); // n_aitken_ = get_field_in("n_aitken").get_view(); + Kokkos::resize(qc_accum_dst_, ncol_, nlev_); // qc_accum_dst_ = get_field_in("qc_accum_dst").get_view(); + Kokkos::resize(qc_accum_so4_, ncol_, nlev_); // qc_accum_so4_ = get_field_in("qc_accum_so4").get_view(); + Kokkos::resize(qc_accum_mom_, ncol_, nlev_); // qc_accum_mom_ = get_field_in("qc_accum_mom").get_view(); + Kokkos::resize(qc_accum_nacl_, ncol_, nlev_); // qc_accum_mom_ = get_field_in("qc_accum_mom").get_view(); + Kokkos::resize(qc_accum_bc_, ncol_, nlev_); // qc_accum_bc_ = get_field_in("qc_accum_bc").get_view(); + Kokkos::resize(qc_accum_pom_, ncol_, nlev_); // qc_accum_pom_ = get_field_in("qc_accum_pom").get_view(); + Kokkos::resize(qc_accum_soa_, ncol_, nlev_); // qc_accum_soa_ = get_field_in("qc_accum_soa").get_view(); + Kokkos::resize(qi_accum_dst_, ncol_, nlev_); // qi_accum_dst_ = get_field_in("qi_accum_dst").get_view(); + Kokkos::resize(qi_accum_so4_, ncol_, nlev_); // qi_accum_so4_ = get_field_in("qi_accum_so4").get_view(); + Kokkos::resize(qi_accum_mom_, ncol_, nlev_); // qi_accum_mom_ = get_field_in("qi_accum_mom").get_view(); + Kokkos::resize(qi_accum_bc_, ncol_, nlev_); // qi_accum_bc_ = get_field_in("qi_accum_bc").get_view(); + Kokkos::resize(qi_accum_pom_, ncol_, nlev_); // qi_accum_pom_ = get_field_in("qi_accum_pom").get_view(); + Kokkos::resize(qi_accum_soa_, ncol_, nlev_); // qi_accum_soa_ = get_field_in("qi_accum_soa").get_view(); + Kokkos::resize(qc_coarse_dst_, ncol_, nlev_); // qc_coarse_dst_ = get_field_in("qc_coarse_dst").get_view(); + Kokkos::resize(qc_coarse_nacl_, ncol_, nlev_); // qc_coarse_nacl_ = get_field_in("qc_coarse_nacl").get_view(); + Kokkos::resize(qc_coarse_mom_, ncol_, nlev_); // ci_coarse_mom_ = get_field_in("qc_coarse_mom").get_view(); + Kokkos::resize(qc_coarse_bc_, ncol_, nlev_); // qc_coarse_bc_ = get_field_in("qc_coarse_bc").get_view(); + Kokkos::resize(qc_coarse_pom_, ncol_, nlev_); // qc_coarse_pom_ = get_field_in("qc_coarse_pom").get_view(); + Kokkos::resize(qc_coarse_soa_, ncol_, nlev_); // qc_coarse_soa_ = get_field_in("qc_coarse_soa").get_view(); + // + qi_coarse_dst_ = get_field_in("qi_coarse_dst").get_view(); + qi_coarse_nacl_ = get_field_in("qi_coarse_nacl").get_view(); + qi_coarse_so4_ = get_field_in("qi_coarse_so4").get_view(); + qi_coarse_mom_ = get_field_in("qi_coarse_mom").get_view(); + qi_coarse_bc_ = get_field_in("qi_coarse_bc").get_view(); + qi_coarse_pom_ = get_field_in("qi_coarse_pom").get_view(); + qi_coarse_soa_ = get_field_in("qi_coarse_soa").get_view(); + + Kokkos::resize(qi_pcarbon_bc_, ncol_, nlev_); // qi_pcarbon_bc_ = get_field_in("qi_pcarbon_bc").get_view(); + Kokkos::resize(qi_pcarbon_mom_, ncol_, nlev_); // qi_pcarbon_mom_ = get_field_in("qi_pcarbon_mom").get_view(); + Kokkos::resize(qi_pcarbon_pom_, ncol_, nlev_); // qi_pcarbon_pom_ = get_field_in("qi_pcarbon_pom").get_view(); + Kokkos::resize(nc_accum_, ncol_, nlev_); // nc_accum_ = get_field_in("nc_accum").get_view(); + Kokkos::resize(ni_accum_, ncol_, nlev_); // ni_accum_ = get_field_in("ni_accum").get_view(); + Kokkos::resize(ni_coarse_, ncol_, nlev_); // ni_coarse_ = get_field_in("ni_coarse").get_view(); + Kokkos::resize(ni_aitken_, ncol_, nlev_); // ni_aitken_ = get_field_in("ni_aitken").get_view(); Kokkos::resize(zm_, ncol_, nlev_); // zm_ = get_field_in("zm").get_view(); Kokkos::resize(state_q_, ncol_, nlev_, mam4::ndrop::nvars); // state_q_ = get_field_in("state_q").get_view(); Kokkos::resize(ncldwtr_, ncol_, nlev_); // ncldwtr_ = get_field_in("ncldwtr").get_view(); Kokkos::resize(cldo_, ncol_, nlev_); // cldo_ = get_field_in("cldo").get_view(); - Kokkos::resize(qqcw_inp_, ncol_, nlev_, mam4::ndrop::ncnst_tot); // qqcw_inp_ = get_field_in("qqcw").get_view(); + qqcw_input_ = get_field_in("qqcw").get_view(); for (int i=0; i(); - dgnum_ = get_field_out("dgnum").get_view(); + cldfrac_ = get_field_in("cldfrac_tot").get_view(); + dgnum_ = get_field_out("dgnum").get_view(); nihf_ = get_field_out("nihf").get_view(); niim_ = get_field_out("niim").get_view(); nidep_ = get_field_out("nidep").get_view(); @@ -282,7 +462,7 @@ void MAMAci::initialize_impl(const RunType run_type) { naai_hom_ = get_field_out("naai_hom").get_view(); naai_ = get_field_out("naai").get_view(); qcld_ = get_field_out("qcld").get_view(); - ptend_q_inp_ = get_field_out("ptend_q").get_view(); + ptend_q_output_ = get_field_out("ptend_q").get_view(); tendnd_ = get_field_out("tendnd").get_view(); factnum_ = get_field_out("factnum").get_view(); ndropcol_ = get_field_out("ndropcol").get_view(); @@ -298,14 +478,18 @@ void MAMAci::initialize_impl(const RunType run_type) { liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); qc_ = get_field_in("qc").get_view(); qi_ = get_field_in("qi").get_view(); + nc_ = get_field_in("nc").get_view(); kvh_ = get_field_in("kvh").get_view(); - // configure the nucleation parameterization - mam4::NucleateIce::Config config; mam4::AeroConfig aero_config; - nucleate_ice_.init(aero_config, config); + // configure the nucleation parameterization + mam4::NucleateIce::Config nucleate_ice_config; + nucleate_ice_.init(aero_config, nucleate_ice_config); + // configure the heterogeneous freezing parameterization + mam4::Hetfrz::Config hetfrz_config; + hetfrz_.init(aero_config, hetfrz_config); /* NOTE: All derived variables (like rpdel and geopotential height) should be computed in preprocess struct @@ -322,41 +506,57 @@ void MAMAci::run_impl(const double dt) { // Negative omega means rising motion //--------------------------------------------------------- - //Get physical constants - using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] - // Alias member variables - auto w0 = w0_; - auto w2 = w2_; auto rho = rho_; auto tke = tke_; auto T_mid = T_mid_; auto p_mid = p_mid_; auto p_int = p_int_; auto pdel = pdel_; - auto w_sec = w_sec_; auto wsub = wsub_; auto wsubice = wsubice_; - auto wsig = wsig_; auto qv_dry = qv_dry_; auto cldfrac = cldfrac_; auto w_updraft = w_updraft_; const auto nlev = nlev_; const auto top_lev = top_lev_; - auto q_coarse_dst = q_coarse_dst_; - auto q_coarse_nacl = q_coarse_nacl_; - auto q_coarse_so4 = q_coarse_so4_; - auto q_coarse_mom = q_coarse_mom_; - auto q_coarse_bc = q_coarse_bc_; - auto q_coarse_pom = q_coarse_pom_; - auto q_coarse_soa = q_coarse_soa_; - auto n_coarse = n_coarse_; - auto n_aitken = n_aitken_; - auto dgnum = dgnum_; + auto qc_accum_dst = qc_accum_dst_; + auto qc_accum_so4 = qc_accum_so4_; + auto qc_accum_mom = qc_accum_mom_; + auto qc_accum_nacl = qc_accum_nacl_; + auto qc_accum_bc = qc_accum_bc_; + auto qc_accum_pom = qc_accum_pom_; + auto qc_accum_soa = qc_accum_soa_; + + auto qi_accum_dst = qi_accum_dst_; + auto qi_accum_so4 = qi_accum_so4_; + auto qi_accum_mom = qi_accum_mom_; + auto qi_accum_bc = qi_accum_bc_; + auto qi_accum_pom = qi_accum_pom_; + auto qi_accum_soa = qi_accum_soa_; + + auto qc_coarse_dst = qc_coarse_dst_; + auto qc_coarse_nacl = qc_coarse_nacl_; + auto qc_coarse_mom = qc_coarse_mom_; + auto qc_coarse_bc = qc_coarse_bc_; + auto qc_coarse_pom = qc_coarse_pom_; + auto qc_coarse_soa = qc_coarse_soa_; + auto qi_coarse_dst = qi_coarse_dst_; + auto qi_coarse_nacl = qi_coarse_nacl_; + auto qi_coarse_so4 = qi_coarse_so4_; + auto qi_coarse_mom = qi_coarse_mom_; + auto qi_coarse_bc = qi_coarse_bc_; + auto qi_coarse_pom = qi_coarse_pom_; + auto qi_coarse_soa = qi_coarse_soa_; + auto qi_pcarbon_bc = qi_pcarbon_bc_; + auto qi_pcarbon_mom = qi_pcarbon_mom_; + auto qi_pcarbon_pom = qi_pcarbon_pom_; + auto nc_accum = nc_accum_; + auto ni_accum = ni_accum_; + auto ni_coarse = ni_coarse_; + auto ni_aitken = ni_aitken_; auto nihf = nihf_; auto niim = niim_; auto nidep = nidep_; @@ -366,6 +566,7 @@ void MAMAci::run_impl(const double dt) { auto liqcldf = liqcldf_; auto qc = qc_; auto qi = qi_; + auto nc = nc_; auto lcldn = lcldn_; auto lcldo = lcldo_; auto aitken_dry_dia = aitken_dry_dia_; @@ -375,7 +576,6 @@ void MAMAci::run_impl(const double dt) { auto ncldwtr = ncldwtr_; auto kvh = kvh_; auto qcld = qcld_; - auto ptend_q_inp = ptend_q_inp_; auto tendnd = tendnd_; auto factnum = factnum_; auto ndropcol = ndropcol_ ; @@ -383,108 +583,53 @@ void MAMAci::run_impl(const double dt) { auto nsource = nsource_ ; auto wtke = wtke_ ; auto cldo = cldo_; - auto qqcw_inp = qqcw_inp_; auto ccn = ccn_; auto coltend_outp = coltend_outp_; auto coltend_cw_outp = coltend_cw_outp_; auto nact = nact_; auto mact = mact_; - auto eddy_diff = scratch_mem_[0]; - auto zn = scratch_mem_[1]; - auto csbot = scratch_mem_[2]; - auto zs = scratch_mem_[3]; - auto overlapp = scratch_mem_[4]; - auto overlapm = scratch_mem_[5]; - auto eddy_diff_kp = scratch_mem_[6]; - auto eddy_diff_km = scratch_mem_[7]; - auto qncld = scratch_mem_[8]; - auto srcn = scratch_mem_[9]; - auto source = scratch_mem_[10]; - auto dz = scratch_mem_[11]; - auto csbot_cscen = scratch_mem_[12]; - auto raertend = scratch_mem_[13]; - auto qqcwtend = scratch_mem_[14]; - - auto team_policy = haero::ThreadTeamPolicy(ncol_, Kokkos::AUTO); - - // NOTE: All the inputs are available to compute w0 - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - for (int i=0; i::template view_1d; view_1d dummy("DummyView", nlev); - - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); - }); - }); - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -519,21 +655,26 @@ void MAMAci::run_impl(const double dt) { mam4::Prognostics progs(nlev); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(q_coarse_dst, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(q_coarse_nacl, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(q_coarse_so4, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(q_coarse_mom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(q_coarse_bc, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(q_coarse_pom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(q_coarse_soa, icol); - progs.n_mode_i[coarse_idx] = ekat::subview(n_coarse, icol); - progs.n_mode_i[aitken_idx] = ekat::subview(n_aitken, icol); + const int dst_idx = static_cast(mam4::AeroId::DST); + + progs.q_aero_i[coarse_idx][dst_idx] = ekat::subview(qi_coarse_dst, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qi_coarse_nacl, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_coarse_so4, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_coarse_mom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_coarse_bc, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_coarse_pom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_coarse_soa, icol); + progs.n_mode_i[coarse_idx] = ekat::subview(ni_coarse, icol); + progs.n_mode_i[aitken_idx] = ekat::subview(ni_aitken, icol); // nucleation doesn't use any diagnostics, so it's okay to leave this alone // for now mam4::Diagnostics diags(nlev); diags.dry_geometric_mean_diameter_i[aitken_idx] = ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated + // in nucleate_ice_.compute_tendencies. diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); diags.icenuc_num_immfrz = ekat::subview(niim, icol); diags.icenuc_num_depnuc = ekat::subview(nidep, icol); @@ -564,6 +705,7 @@ void MAMAci::run_impl(const double dt) { // is above qsmall threshold value //------------------------------------------------------------- + const Real dtmicro = dtmicro_; static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) /* @@ -683,7 +825,7 @@ void MAMAci::run_impl(const double dt) { mam4::ndrop::dropmixnuc( - team, dtmicro, + team, dtmicro, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), @@ -737,7 +879,7 @@ void MAMAci::run_impl(const double dt) { const int icol = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { for (int i=0; i(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + const int pcarbon_idx = static_cast(mam4::ModeIndex::PrimaryCarbon); + + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qc_accum_bc, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qc_accum_dst, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qc_accum_mom, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qc_accum_pom, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qc_accum_nacl, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qc_accum_so4, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qc_accum_soa, icol); + + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_accum_bc, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qi_accum_dst, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_accum_mom, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_accum_pom, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_accum_so4, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_accum_soa, icol); + + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qc_coarse_bc, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qc_coarse_dst, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qc_coarse_mom, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qc_coarse_nacl, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qc_coarse_pom, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qc_coarse_soa, icol); + + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_coarse_bc, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qi_coarse_dst, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_coarse_mom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qi_coarse_nacl, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_coarse_pom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_coarse_so4, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_coarse_soa, icol); + + progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_pcarbon_bc, icol); + progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_pcarbon_mom, icol); + progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_pcarbon_pom, icol); + + progs.n_mode_c[accum_idx ] = ekat::subview(nc_accum, icol); + + progs.n_mode_i[accum_idx ] = ekat::subview(ni_accum, icol); + progs.n_mode_i[coarse_idx] = ekat::subview(ni_coarse, icol); + + mam4::Diagnostics diags(nlev); + diags.stratiform_cloud_fraction = ekat::subview(stratiform_cloud_fraction, icol); + diags.activation_fraction[accum_idx] = ekat::subview(activation_fraction_accum_idx, icol); + diags.activation_fraction[coarse_idx] = ekat::subview(activation_fraction_coarse_idx, icol); + + diags.hetfrz_immersion_nucleation_tend = ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t=0, dt=0; + hetfrz_.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); + + }); + /* call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, & ! in qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index c130ff06d86..c5ed6a5a90c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -15,10 +15,12 @@ namespace scream class MAMAci final : public scream::AtmosphereProcess { +public: using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; + mam4::Hetfrz hetfrz_; // views for single- and multi-column data using view_1d = typename KT::template view_1d; @@ -33,6 +35,7 @@ class MAMAci final : public scream::AtmosphereProcess { // FIXME the time step for microphysics [s] need to get from the input const Real dtmicro_ = .0001; +private: // rho is air density [kg/m3] view_2d rho_; @@ -43,21 +46,48 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d tke_; const_view_2d qv_dry_; - view_2d cldfrac_; - //const_view_2d w_updraft_; - view_2d w_updraft_; + const_view_2d cldfrac_; + const_view_2d w_updraft_; view_2d aitken_dry_dia_; - view_2d q_coarse_dst_; - view_2d q_coarse_nacl_; - view_2d q_coarse_so4_; - view_2d q_coarse_mom_; - view_2d q_coarse_bc_; - view_2d q_coarse_pom_; - view_2d q_coarse_soa_; - view_2d n_coarse_; - view_2d n_aitken_; - view_3d dgnum_; + view_2d qc_coarse_bc_; + view_2d qc_coarse_dst_; + view_2d qc_coarse_mom_; + view_2d qc_coarse_nacl_; + view_2d qc_coarse_pom_; + view_2d qc_coarse_soa_; + + view_2d qc_accum_bc_; + view_2d qc_accum_dst_; + view_2d qc_accum_mom_; + view_2d qc_accum_nacl_; + view_2d qc_accum_pom_; + view_2d qc_accum_so4_; + view_2d qc_accum_soa_; + + view_2d qi_accum_dst_; + view_2d qi_accum_so4_; + view_2d qi_accum_mom_; + view_2d qi_accum_bc_; + view_2d qi_accum_pom_; + view_2d qi_accum_soa_; + + view_2d qi_coarse_dst_; + view_2d qi_coarse_nacl_; + view_2d qi_coarse_so4_; + view_2d qi_coarse_mom_; + view_2d qi_coarse_bc_; + view_2d qi_coarse_pom_; + view_2d qi_coarse_soa_; + + view_2d qi_pcarbon_bc_; + view_2d qi_pcarbon_mom_; + view_2d qi_pcarbon_pom_; + view_2d nc_accum_; + view_2d ni_accum_; + view_2d ni_coarse_; + view_2d ni_aitken_; + const_view_3d dgnum_; view_2d nihf_; view_2d niim_; view_2d nidep_; @@ -67,6 +97,7 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d qc_; const_view_2d qi_; + const_view_2d nc_; const_view_2d kvh_; view_2d cldo_; @@ -79,9 +110,9 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d qcld_; view_2d tendnd_; view_2d ptend_q_[mam4::ndrop::nvar_ptend_q]; - view_3d ptend_q_inp_; + view_3d ptend_q_output_; view_3d factnum_; - view_3d qqcw_inp_; + const_view_3d qqcw_input_; view_2d qqcw_[mam4::ndrop::ncnst_tot]; view_2d ndropcol_; view_2d ndropmix_; @@ -96,8 +127,16 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d raercol_[mam4::ndrop::pver][2]; view_3d nact_; view_3d mact_; - view_2d scratch_mem_[15]; + view_2d dropmixnuc_scratch_mem_[15]; + + view_2d stratiform_cloud_fraction_; + view_2d activation_fraction_accum_idx_; + view_2d activation_fraction_coarse_idx_; + view_2d hetfrz_immersion_nucleation_tend_; + view_2d hetfrz_contact_nucleation_tend_; + view_2d hetfrz_depostion_nucleation_tend_; + view_2d diagnostic_scratch_[42]; // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; From 83ba683df32baa756e6797c420b8e341218f5a9c Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 25 Jan 2024 15:30:34 -0700 Subject: [PATCH 019/142] General code clean up by using structs from mam_coupling.hpp --- .../mam/eamxx_mam_aci_process_interface.cpp | 1329 ++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 107 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 1 + .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 46 +- 4 files changed, 707 insertions(+), 776 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 1fb52659ac5..91de9d4a03a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -28,9 +28,8 @@ void compute_w0_and_rho( haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, MAMAci::view_2d rho, - MAMAci::const_view_2d p_mid, - MAMAci::const_view_2d T_mid, - MAMAci::const_view_2d omega, + mam_coupling::WetAtmosphere &wet_atmosphere, + mam_coupling::DryAtmosphere &dry_atmosphere, const int top_lev, const int nlev) { @@ -38,6 +37,9 @@ void compute_w0_and_rho( using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + MAMAci::const_view_2d omega = wet_atmosphere.omega; + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { @@ -175,466 +177,28 @@ void compute_aitken_dry_diameter( } void compute_nucleate_ice_tendencies( + const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, - const int nlev) + MAMAci::view_2d nihf, + MAMAci::view_2d niim, + MAMAci::view_2d nidep, + MAMAci::view_2d nimey, + MAMAci::view_2d naai_hom, + MAMAci::view_2d naai, + mam_coupling::AerosolState &aerosol_state, + mam_coupling::DryAtmosphere &dry_atmosphere, + MAMAci::view_2d aitken_dry_dia, + const int nlev) { -} - -} - - -//FIXME: The following variables are namelist variables -const Real wsubmin = 1; - -MAMAci::MAMAci( - const ekat::Comm& comm, - const ekat::ParameterList& params) - : AtmosphereProcess(comm, params){ -} - - -//Return type of the process -AtmosphereProcessType MAMAci::type() const{ - return AtmosphereProcessType::Physics; -} - -//return name of the process -std::string MAMAci::name() const{ - return "mam4_aci"; - } - -//set grid for all the inputs and outputs -void MAMAci::set_grids(const std::shared_ptr grids_manager) { - m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI set grid"); - - grid_ = grids_manager->get_grid("Physics"); //Use physics grid - const auto& grid_name = grid_->name(); - - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - - Kokkos::resize(rho_, ncol_, nlev_); - Kokkos::resize(w0_, ncol_, nlev_); - Kokkos::resize(tke_, ncol_, nlev_+1); - Kokkos::resize(wsub_, ncol_, nlev_); - Kokkos::resize(wsubice_, ncol_, nlev_); - Kokkos::resize(wsig_, ncol_, nlev_); - Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(lcldn_, ncol_, nlev_); - Kokkos::resize(lcldo_, ncol_, nlev_); - Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); - Kokkos::resize(rpdel_, ncol_, nlev_); - - for (int i=0; i<15; ++i) { - Kokkos::resize(dropmixnuc_scratch_mem_[i],ncol_, nlev_); - } - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; - - // Layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points - FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces - - using namespace ekat::units; - auto q_unit = kg/kg; // units of mass mixing ratios of tracers - q_unit.set_string("kg/kg"); - auto n_unit = 1/kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); - auto nondim = Units::nondimensional(); - // atmospheric quantities - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] - add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints - add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints - add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces - add_field("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // - // - add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - - - // FIXME: These should all come from the intput file. Might need to be added to the input file? - add_field("qi_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_so4", scalar3d_layout_mid, q_unit, grid_name); // Sulfuric Acid mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] - //add_field("qi_accum_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for accum mode [kg/kg] - //add_field("qi_accum_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for accum mode [kg/kg] - //add_field("qi_accum_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for accum mode [kg/kg] - //add_field("qi_accumn_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for primary carbon mode [kg/kg] - //add_field("qi_accumn_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for primary carbon mode [kg/kg]] - //add_field("qi_accumn_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for primary carbon mode [kg/kg]kg] - //add_field("qi_pcarbon_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for primary carbon mode [kg/kg] - //add_field("qi_pcarbon_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for primary carbon mode [kg/kg]] - //add_field("qi_pcarbon_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for primary carbon mode [kg/kg]kg] - //add_field("ni_accum", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] - //add_field("ni_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] - //add_field("ni_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] - //add_field("zm", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) - //add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] - //add_field("ncldwtr", , n_unit, grid_name); // initial droplet number mixing ratio [#/kg] - //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] - // - add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] - add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] - - - add_field("nihf",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] - add_field("niim",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] - add_field("nidep",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] - add_field("nimey",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] - add_field("naai_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] - add_field("naai",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] - add_field("qcld",scalar3d_layout_mid , n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] - add_field("ptend_q", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne aerosols [#/kg] - add_field("tendnd",scalar3d_layout_mid , n_unit/s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] - add_field("factnum", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::AeroConfig::num_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] - add_field("ndropcol",scalar3d_layout_mid , n_unit/s, grid_name); // - add_field("ndropmix",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio tendency due to mixing [#/kg/s] - add_field("nsource",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] - add_field("wtke",scalar3d_layout_mid , n_unit/s, grid_name); // - add_field("ccn", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, grid_name); //number conc of aerosols activated at supersat [#/m^3] - // note: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud [#/m^2/s]] - // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] - add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output - add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency - - - //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - //interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - - //cloudborne aerosol tracers of interest: number (n) mixing ratios - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } - - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { - //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); - } - } - } - - //Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM - //These outputs should come from the cloud macrophysics process (e.g., SHOC) - auto m2 = m*m; - m2.set_string("m^2"); - auto s2 = s*s; - s2.set_string("s^2"); - - //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time - add_field("w_sec", scalar3d_layout_int, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints - - add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints - add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints - add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat - - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables - num_aero_modes_ = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes_} }; // mid points - add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols - - - /*NOTE on other inputs for the aci process: - 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density - 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using - pseudo_density, p_mid, T_mid and qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). - qv_mid can be obtained from "get_field_in" call*/ - -} - -void MAMAci::initialize_impl(const RunType run_type) { - m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); - /* - NOTE: All other inputs should follow the way "pseudo_density" is initialized - */ - // set atmosphere state data - pdel_ = get_field_in("pseudo_density").get_view(); - omega_ = get_field_in("omega").get_view(); - p_mid_ = get_field_in("p_mid").get_view(); - p_int_ = get_field_in("p_int").get_view(); - T_mid_ = get_field_in("T_mid").get_view(); - w_sec_ = get_field_in("w_sec").get_view(); - qv_dry_ = get_field_in("qv").get_view(); - w_updraft_ = get_field_in("w_updraft").get_view(); // updraft velocity [m/s] - // - stratiform_cloud_fraction_ = get_field_out("stratiform_cloud_fraction").get_view(); - activation_fraction_accum_idx_ = get_field_out("activation_fraction_accum").get_view(); - activation_fraction_coarse_idx_ = get_field_out("activation_fraction_coarse").get_view(); - - // FIXME: These should all come from the intput file. Might need to be added to the input file? - Kokkos::resize(qc_accum_dst_, ncol_, nlev_); // qc_accum_dst_ = get_field_in("qc_accum_dst").get_view(); - Kokkos::resize(qc_accum_so4_, ncol_, nlev_); // qc_accum_so4_ = get_field_in("qc_accum_so4").get_view(); - Kokkos::resize(qc_accum_mom_, ncol_, nlev_); // qc_accum_mom_ = get_field_in("qc_accum_mom").get_view(); - Kokkos::resize(qc_accum_nacl_, ncol_, nlev_); // qc_accum_mom_ = get_field_in("qc_accum_mom").get_view(); - Kokkos::resize(qc_accum_bc_, ncol_, nlev_); // qc_accum_bc_ = get_field_in("qc_accum_bc").get_view(); - Kokkos::resize(qc_accum_pom_, ncol_, nlev_); // qc_accum_pom_ = get_field_in("qc_accum_pom").get_view(); - Kokkos::resize(qc_accum_soa_, ncol_, nlev_); // qc_accum_soa_ = get_field_in("qc_accum_soa").get_view(); - Kokkos::resize(qi_accum_dst_, ncol_, nlev_); // qi_accum_dst_ = get_field_in("qi_accum_dst").get_view(); - Kokkos::resize(qi_accum_so4_, ncol_, nlev_); // qi_accum_so4_ = get_field_in("qi_accum_so4").get_view(); - Kokkos::resize(qi_accum_mom_, ncol_, nlev_); // qi_accum_mom_ = get_field_in("qi_accum_mom").get_view(); - Kokkos::resize(qi_accum_bc_, ncol_, nlev_); // qi_accum_bc_ = get_field_in("qi_accum_bc").get_view(); - Kokkos::resize(qi_accum_pom_, ncol_, nlev_); // qi_accum_pom_ = get_field_in("qi_accum_pom").get_view(); - Kokkos::resize(qi_accum_soa_, ncol_, nlev_); // qi_accum_soa_ = get_field_in("qi_accum_soa").get_view(); - Kokkos::resize(qc_coarse_dst_, ncol_, nlev_); // qc_coarse_dst_ = get_field_in("qc_coarse_dst").get_view(); - Kokkos::resize(qc_coarse_nacl_, ncol_, nlev_); // qc_coarse_nacl_ = get_field_in("qc_coarse_nacl").get_view(); - Kokkos::resize(qc_coarse_mom_, ncol_, nlev_); // ci_coarse_mom_ = get_field_in("qc_coarse_mom").get_view(); - Kokkos::resize(qc_coarse_bc_, ncol_, nlev_); // qc_coarse_bc_ = get_field_in("qc_coarse_bc").get_view(); - Kokkos::resize(qc_coarse_pom_, ncol_, nlev_); // qc_coarse_pom_ = get_field_in("qc_coarse_pom").get_view(); - Kokkos::resize(qc_coarse_soa_, ncol_, nlev_); // qc_coarse_soa_ = get_field_in("qc_coarse_soa").get_view(); - // - qi_coarse_dst_ = get_field_in("qi_coarse_dst").get_view(); - qi_coarse_nacl_ = get_field_in("qi_coarse_nacl").get_view(); - qi_coarse_so4_ = get_field_in("qi_coarse_so4").get_view(); - qi_coarse_mom_ = get_field_in("qi_coarse_mom").get_view(); - qi_coarse_bc_ = get_field_in("qi_coarse_bc").get_view(); - qi_coarse_pom_ = get_field_in("qi_coarse_pom").get_view(); - qi_coarse_soa_ = get_field_in("qi_coarse_soa").get_view(); - - Kokkos::resize(qi_pcarbon_bc_, ncol_, nlev_); // qi_pcarbon_bc_ = get_field_in("qi_pcarbon_bc").get_view(); - Kokkos::resize(qi_pcarbon_mom_, ncol_, nlev_); // qi_pcarbon_mom_ = get_field_in("qi_pcarbon_mom").get_view(); - Kokkos::resize(qi_pcarbon_pom_, ncol_, nlev_); // qi_pcarbon_pom_ = get_field_in("qi_pcarbon_pom").get_view(); - Kokkos::resize(nc_accum_, ncol_, nlev_); // nc_accum_ = get_field_in("nc_accum").get_view(); - Kokkos::resize(ni_accum_, ncol_, nlev_); // ni_accum_ = get_field_in("ni_accum").get_view(); - Kokkos::resize(ni_coarse_, ncol_, nlev_); // ni_coarse_ = get_field_in("ni_coarse").get_view(); - Kokkos::resize(ni_aitken_, ncol_, nlev_); // ni_aitken_ = get_field_in("ni_aitken").get_view(); - Kokkos::resize(zm_, ncol_, nlev_); // zm_ = get_field_in("zm").get_view(); - Kokkos::resize(state_q_, ncol_, nlev_, mam4::ndrop::nvars); // state_q_ = get_field_in("state_q").get_view(); - Kokkos::resize(ncldwtr_, ncol_, nlev_); // ncldwtr_ = get_field_in("ncldwtr").get_view(); - Kokkos::resize(cldo_, ncol_, nlev_); // cldo_ = get_field_in("cldo").get_view(); - qqcw_input_ = get_field_in("qqcw").get_view(); - - for (int i=0; i(); - dgnum_ = get_field_out("dgnum").get_view(); - nihf_ = get_field_out("nihf").get_view(); - niim_ = get_field_out("niim").get_view(); - nidep_ = get_field_out("nidep").get_view(); - nimey_ = get_field_out("nimey").get_view(); - naai_hom_ = get_field_out("naai_hom").get_view(); - naai_ = get_field_out("naai").get_view(); - qcld_ = get_field_out("qcld").get_view(); - ptend_q_output_ = get_field_out("ptend_q").get_view(); - tendnd_ = get_field_out("tendnd").get_view(); - factnum_ = get_field_out("factnum").get_view(); - ndropcol_ = get_field_out("ndropcol").get_view(); - ndropmix_ = get_field_out("ndropmix").get_view(); - nsource_ = get_field_out("nsource").get_view(); - wtke_ = get_field_out("wtke").get_view(); - ccn_ = get_field_out("ccn").get_view(); - coltend_outp_ = get_field_out("coltend").get_view(); - coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - - - - liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); - qc_ = get_field_in("qc").get_view(); - qi_ = get_field_in("qi").get_view(); - nc_ = get_field_in("nc").get_view(); - kvh_ = get_field_in("kvh").get_view(); - - - mam4::AeroConfig aero_config; - // configure the nucleation parameterization - mam4::NucleateIce::Config nucleate_ice_config; - nucleate_ice_.init(aero_config, nucleate_ice_config); - - // configure the heterogeneous freezing parameterization - mam4::Hetfrz::Config hetfrz_config; - hetfrz_.init(aero_config, hetfrz_config); - /* - NOTE: All derived variables (like rpdel and geopotential height) should be computed in - preprocess struct - */ - //preprocess_.set_variables(pdel_); - -} - -void MAMAci::run_impl(const double dt) { - m_atm_logger->log(ekat::logger::LogLevel::info,"calling ACI run"); - - //---------------------------------------------------------- - // Convert from omega to w (vertical velocity) - // Negative omega means rising motion - //--------------------------------------------------------- - - - // Alias member variables - auto rho = rho_; - auto tke = tke_; - auto T_mid = T_mid_; - auto p_mid = p_mid_; - auto p_int = p_int_; - auto pdel = pdel_; - auto wsub = wsub_; - auto wsubice = wsubice_; - auto qv_dry = qv_dry_; - auto cldfrac = cldfrac_; - auto w_updraft = w_updraft_; - const auto nlev = nlev_; - const auto top_lev = top_lev_; - - auto qc_accum_dst = qc_accum_dst_; - auto qc_accum_so4 = qc_accum_so4_; - auto qc_accum_mom = qc_accum_mom_; - auto qc_accum_nacl = qc_accum_nacl_; - auto qc_accum_bc = qc_accum_bc_; - auto qc_accum_pom = qc_accum_pom_; - auto qc_accum_soa = qc_accum_soa_; - - auto qi_accum_dst = qi_accum_dst_; - auto qi_accum_so4 = qi_accum_so4_; - auto qi_accum_mom = qi_accum_mom_; - auto qi_accum_bc = qi_accum_bc_; - auto qi_accum_pom = qi_accum_pom_; - auto qi_accum_soa = qi_accum_soa_; - - auto qc_coarse_dst = qc_coarse_dst_; - auto qc_coarse_nacl = qc_coarse_nacl_; - auto qc_coarse_mom = qc_coarse_mom_; - auto qc_coarse_bc = qc_coarse_bc_; - auto qc_coarse_pom = qc_coarse_pom_; - auto qc_coarse_soa = qc_coarse_soa_; - auto qi_coarse_dst = qi_coarse_dst_; - auto qi_coarse_nacl = qi_coarse_nacl_; - auto qi_coarse_so4 = qi_coarse_so4_; - auto qi_coarse_mom = qi_coarse_mom_; - auto qi_coarse_bc = qi_coarse_bc_; - auto qi_coarse_pom = qi_coarse_pom_; - auto qi_coarse_soa = qi_coarse_soa_; - auto qi_pcarbon_bc = qi_pcarbon_bc_; - auto qi_pcarbon_mom = qi_pcarbon_mom_; - auto qi_pcarbon_pom = qi_pcarbon_pom_; - auto nc_accum = nc_accum_; - auto ni_accum = ni_accum_; - auto ni_coarse = ni_coarse_; - auto ni_aitken = ni_aitken_; - auto nihf = nihf_; - auto niim = niim_; - auto nidep = nidep_; - auto nimey = nimey_; - auto naai_hom = naai_hom_; - auto naai = naai_; - auto liqcldf = liqcldf_; - auto qc = qc_; - auto qi = qi_; - auto nc = nc_; - auto lcldn = lcldn_; - auto lcldo = lcldo_; - auto aitken_dry_dia = aitken_dry_dia_; - auto rpdel = rpdel_; - auto zm = zm_; - auto state_q = state_q_; - auto ncldwtr = ncldwtr_; - auto kvh = kvh_; - auto qcld = qcld_; - auto tendnd = tendnd_; - auto factnum = factnum_; - auto ndropcol = ndropcol_ ; - auto ndropmix = ndropmix_ ; - auto nsource = nsource_ ; - auto wtke = wtke_ ; - auto cldo = cldo_; - auto ccn = ccn_; - auto coltend_outp = coltend_outp_; - auto coltend_cw_outp = coltend_cw_outp_; - auto nact = nact_; - auto mact = mact_; - auto eddy_diff = dropmixnuc_scratch_mem_[0]; - auto zn = dropmixnuc_scratch_mem_[1]; - auto csbot = dropmixnuc_scratch_mem_[2]; - auto zs = dropmixnuc_scratch_mem_[3]; - auto overlapp = dropmixnuc_scratch_mem_[4]; - auto overlapm = dropmixnuc_scratch_mem_[5]; - auto eddy_diff_kp = dropmixnuc_scratch_mem_[6]; - auto eddy_diff_km = dropmixnuc_scratch_mem_[7]; - auto qncld = dropmixnuc_scratch_mem_[8]; - auto srcn = dropmixnuc_scratch_mem_[9]; - auto source = dropmixnuc_scratch_mem_[10]; - auto dz = dropmixnuc_scratch_mem_[11]; - auto csbot_cscen = dropmixnuc_scratch_mem_[12]; - auto raertend = dropmixnuc_scratch_mem_[13]; - auto qqcwtend = dropmixnuc_scratch_mem_[14]; - - auto hetfrz_immersion_nucleation_tend = hetfrz_immersion_nucleation_tend_; - auto hetfrz_contact_nucleation_tend = hetfrz_contact_nucleation_tend_; - auto hetfrz_depostion_nucleation_tend = hetfrz_depostion_nucleation_tend_; - auto stratiform_cloud_fraction = stratiform_cloud_fraction_; - auto activation_fraction_accum_idx = activation_fraction_accum_idx_; - auto activation_fraction_coarse_idx = activation_fraction_coarse_idx_; - auto diagnostic_scratch = diagnostic_scratch_; - - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - - copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_, mam4::ndrop::ncnst_tot); - - // All the inputs are available to compute w0 and rho - compute_w0_and_rho(team_policy, w0_, rho_, p_mid_, T_mid_, omega_, top_lev_, nlev_); - - compute_tke_using_w_sec(team_policy, tke_, w_sec_, nlev_); - - compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, wsubmin, top_lev_, nlev_); - - compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); - - compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); - - compute_nucleate_ice_tendencies(team_policy, nlev_); - //T_mid_, p_mid_, qv_dry_, cldfrac_, w_updraft_ - // qi_coarse_dst_, qi_coarse_nacl_, qi_coarse_so4_, qi_coarse_mom_, qi_coarse_bc_, qi_coarse_pom_, qi_coarse_soa_, - //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation //------------------------------------------------------------- + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + MAMAci::const_view_2d qv_dry = dry_atmosphere.qv; + MAMAci::const_view_2d cldfrac = dry_atmosphere.cldfrac; + MAMAci::const_view_2d w_updraft = dry_atmosphere.w_updraft; using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -652,29 +216,17 @@ void MAMAci::run_impl(const double dt) { ekat::subview(w_updraft, icol), pblh); // set surface state data haero::Surface surf{}; - mam4::Prognostics progs(nlev); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - const int dst_idx = static_cast(mam4::AeroId::DST); - - progs.q_aero_i[coarse_idx][dst_idx] = ekat::subview(qi_coarse_dst, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qi_coarse_nacl, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_coarse_so4, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_coarse_mom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_coarse_bc, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_coarse_pom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_coarse_soa, icol); - progs.n_mode_i[coarse_idx] = ekat::subview(ni_coarse, icol); - progs.n_mode_i[aitken_idx] = ekat::subview(ni_aitken, icol); + mam4::Prognostics progs = mam_coupling::aerosols_for_column(aerosol_state, icol); // nucleation doesn't use any diagnostics, so it's okay to leave this alone // for now mam4::Diagnostics diags(nlev); + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); diags.dry_geometric_mean_diameter_i[aitken_idx] = ekat::subview(aitken_dry_dia, icol); // These are the fields that are updated. Taking subviews means that // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated - // in nucleate_ice_.compute_tendencies. + // in nucleate_ice.compute_tendencies. diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); diags.icenuc_num_immfrz = ekat::subview(niim, icol); diags.icenuc_num_depnuc = ekat::subview(nidep, icol); @@ -688,8 +240,6 @@ void MAMAci::run_impl(const double dt) { const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t=0, dt=0; - nucleate_ice_.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); - /* NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". Only output we care about is "naai", "naai_hom" is never used anywhere @@ -699,196 +249,162 @@ void MAMAci::run_impl(const double dt) { rho, wsubice, strat_cld_frac, dgnum, & ! input naai, naai_hom) ! output */ + nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); }); - //------------------------------------------------------------- - // Get old and new liquid cloud fractions when amount of cloud - // is above qsmall threshold value - //------------------------------------------------------------- - - const Real dtmicro = dtmicro_; - static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) - - /* - MUST FIXME NOTE: We need old and new liquid cloud fractions here. - We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to - store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make - a note of it and use the new cloud fraction for the - old cloud fraction. - */ +} +void store_liquid_cloud_fraction( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::const_view_2d liqcldf, + const int top_lev) +{ + //------------------------------------------------------------- + // Get old and new liquid cloud fractions when amount of cloud + // is above qsmall threshold value + + // MUST FIXME NOTE: We need old and new liquid cloud fractions here. + // We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to + // store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make + // a note of it and use the new cloud fraction for the + // old cloud fraction. + //------------------------------------------------------------- + MAMAci::const_view_2d qc = wet_atmosphere.qc; + MAMAci::const_view_2d qi = wet_atmosphere.qi; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); + static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { const Real qcld = qc(icol,kk) + qi(icol,kk); if (qcld > qsmall) { - lcldn(icol,kk)=liqcldf(icol,kk); - lcldo(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old + cloud_frac_new(icol,kk)=liqcldf(icol,kk); + cloud_frac_old(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old + } else { + cloud_frac_new(icol,kk)=0; + cloud_frac_old(icol,kk)=0; } }); }); - /* - Fortran code: - liqcldf(:ncol,:pver) = liq_strat_cld_frac(:ncol,:pver) - lcldn = 0._r8 - lcldo = 0._r8 - do kk = top_lev, pver - do icol = 1, ncol - qcld = qc(icol,kk) + qi(icol,kk) - if (qcld > qsmall) then - lcldn(icol,kk)=liqcldf(icol,kk) - lcldo(icol,kk)=liqcldfo(icol,kk) - end if - end do - end do - */ - - //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous - // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne - // aerosols, not all the cloud borne aerosols. - //------------------------------------------------------------- +} +void compute_recipical_pseudo_density( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d pdel, + const int nlev) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + rpdel(icol,kk) = 1/pdel(icol,kk); + }); + }); +} - /*NOTE: We probably need to store indices for the select few - cloud borne aerosols - - Fortran code: - lchnk_zb = lchnk - begchunk - ! save copy of cloud borne aerosols for use in heterogeneous freezing before - !we change it in dropmixnuc - do ispec = 1, ncnst - call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) - aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) - aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * rho(:ncol,:) - enddo - */ +void call_function_dropmixnuc( + haero::ThreadTeamPolicy team_policy, + mam_coupling::DryAtmosphere &dry_atmosphere, + const Real dtmicro, + MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], + MAMAci::view_2d raercol[mam4::ndrop::pver][2], + MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], + MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], + MAMAci::const_view_2d p_int, + MAMAci::const_view_2d pdel, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d zm, + MAMAci::const_view_3d state_q, + MAMAci::const_view_2d ncldwtr, + MAMAci::const_view_2d kvh, + MAMAci::view_2d qcld, + MAMAci::view_2d wsub, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + MAMAci::view_2d tendnd, + MAMAci::view_3d factnum, + MAMAci::view_2d ndropcol, + MAMAci::view_2d ndropmix, + MAMAci::view_2d nsource, + MAMAci::view_2d wtke, + MAMAci::view_3d ccn, + MAMAci::view_3d nact, + MAMAci::view_3d mact, + MAMAci::view_2d dropmixnuc_scratch_mem[15], + const int nlev) +{ + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); - //------------------------------------------------------------- - // Compute activated fraction of aerosols - //------------------------------------------------------------- - const int ntot_amode = mam4::AeroConfig::num_modes(); + // Initialize the ndrop class. + const int ntot_amode = mam_coupling::num_aero_modes(); const int maxd_aspectype = mam4::ndrop::maxd_aspectype; const int nspec_max = mam4::ndrop::nspec_max; - int nspec_amode[ntot_amode]; - int lspectype_amode[maxd_aspectype][ntot_amode]; - int lmassptr_amode[maxd_aspectype][ntot_amode]; - Real specdens_amode[maxd_aspectype]; - Real spechygro[maxd_aspectype]; - int numptr_amode[ntot_amode]; - int mam_idx[ntot_amode][nspec_max]; - int mam_cnst_idx[ntot_amode][nspec_max]; + int nspec_amode[ntot_amode] = {}; + int lspectype_amode[maxd_aspectype][ntot_amode] = {}; + int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; + Real specdens_amode[maxd_aspectype] = {}; + Real spechygro[maxd_aspectype] = {}; + int numptr_amode[ntot_amode] = {}; + int mam_idx[ntot_amode][nspec_max] = {}; + int mam_cnst_idx[ntot_amode][nspec_max] = {}; + Real exp45logsig[ntot_amode] = {}, + alogsig[ntot_amode] = {}, + num2vol_ratio_min_nmodes[ntot_amode] = {}, + num2vol_ratio_max_nmodes[ntot_amode] = {}; + Real aten = 0; + mam4::ndrop::get_e3sm_parameters( nspec_amode, lspectype_amode, lmassptr_amode, numptr_amode, specdens_amode, spechygro, mam_idx, mam_cnst_idx); - Real exp45logsig[ntot_amode], - alogsig[ntot_amode], - num2vol_ratio_min_nmodes[ntot_amode], - num2vol_ratio_max_nmodes[ntot_amode] = {}; - Real aten = 0; mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, // voltonumbhi_amode num2vol_ratio_max_nmodes); // voltonumblo_amode - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - rpdel(icol,kk) = 1/pdel(icol,kk); - }); - }); - - - - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - mam4::ndrop::View1D raercol_cw[mam4::ndrop::pver][2]; - mam4::ndrop::View1D raercol[mam4::ndrop::pver][2]; - for (int i=0; i::template view_1d; + view_1d dummy("DummyView", nlev); + MAMAci::const_view_2d qc = wet_atmosphere.qc; + MAMAci::const_view_2d nc = wet_atmosphere.nc; + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -925,55 +524,18 @@ void MAMAci::run_impl(const double dt) { pblh); // set surface state data haero::Surface surf{}; - mam4::Prognostics progs(nlev); + mam4::Prognostics progs = mam_coupling::aerosols_for_column(aerosol_state, icol); + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - const int pcarbon_idx = static_cast(mam4::ModeIndex::PrimaryCarbon); - - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qc_accum_bc, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qc_accum_dst, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qc_accum_mom, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qc_accum_pom, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qc_accum_nacl, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qc_accum_so4, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qc_accum_soa, icol); - - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_accum_bc, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qi_accum_dst, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_accum_mom, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_accum_pom, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_accum_so4, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_accum_soa, icol); - - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qc_coarse_bc, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qc_coarse_dst, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qc_coarse_mom, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qc_coarse_nacl, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qc_coarse_pom, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qc_coarse_soa, icol); - - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_coarse_bc, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qi_coarse_dst, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_coarse_mom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qi_coarse_nacl, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_coarse_pom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_coarse_so4, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_coarse_soa, icol); - - progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_pcarbon_bc, icol); - progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_pcarbon_mom, icol); - progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_pcarbon_pom, icol); - - progs.n_mode_c[accum_idx ] = ekat::subview(nc_accum, icol); - - progs.n_mode_i[accum_idx ] = ekat::subview(ni_accum, icol); - progs.n_mode_i[coarse_idx] = ekat::subview(ni_coarse, icol); mam4::Diagnostics diags(nlev); diags.stratiform_cloud_fraction = ekat::subview(stratiform_cloud_fraction, icol); diags.activation_fraction[accum_idx] = ekat::subview(activation_fraction_accum_idx, icol); diags.activation_fraction[coarse_idx] = ekat::subview(activation_fraction_coarse_idx, icol); + // These are the output tendencies from heterogeneous freezing that need to be + // added correctly to the cloud-micorphysics scheme. diags.hetfrz_immersion_nucleation_tend = ekat::subview(hetfrz_immersion_nucleation_tend, icol); diags.hetfrz_contact_nucleation_tend = ekat::subview(hetfrz_contact_nucleation_tend, icol); diags.hetfrz_depostion_nucleation_tend = ekat::subview(hetfrz_depostion_nucleation_tend, icol); @@ -1025,20 +587,393 @@ void MAMAci::run_impl(const double dt) { diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // // grab views from the buffer to store tendencies, not used as all values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t=0, dt=0; - hetfrz_.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); - + hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); }); - - /* - call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, & ! in - qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in - frzimm, frzcnt, frzdep) - */ +} +} + +//FIXME: The following variables are namelist variables +const Real wsubmin = 1; + +MAMAci::MAMAci( + const ekat::Comm& comm, + const ekat::ParameterList& params) + : AtmosphereProcess(comm, params){ +} + + +//Return type of the process +AtmosphereProcessType MAMAci::type() const{ + return AtmosphereProcessType::Physics; +} + +//return name of the process +std::string MAMAci::name() const{ + return "mam4_aci"; + } + +//set grid for all the inputs and outputs +void MAMAci::set_grids(const std::shared_ptr grids_manager) { + m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI set grid"); + + grid_ = grids_manager->get_grid("Physics"); //Use physics grid + const auto& grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + Kokkos::resize(tke_, ncol_, nlev_+1); + Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(wsubice_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(cloud_frac_new_, ncol_, nlev_); + Kokkos::resize(cloud_frac_old_, ncol_, nlev_); + Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); + Kokkos::resize(rpdel_, ncol_, nlev_); + + for (int i=0; i<15; ++i) { + Kokkos::resize(dropmixnuc_scratch_mem_[i],ncol_, nlev_); + } + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variables + FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points + FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces + + using namespace ekat::units; + auto q_unit = kg/kg; // units of mass mixing ratios of tracers + q_unit.set_string("kg/kg"); + auto n_unit = 1/kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + auto nondim = Units::nondimensional(); + // atmospheric quantities + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints + add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints + add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces + add_field ("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + // + // + add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + + add_field ("z_mid", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) + add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] + add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] + //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] + // + add_field ("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] + add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] + + add_field("icenuc_num_hetfrz",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] + add_field("icenuc_num_immfrz",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] + add_field("icenuc_num_depnuc",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] + add_field("icenuc_num_meydep",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + add_field("num_act_aerosol_ice_nucle_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] + add_field("num_act_aerosol_ice_nucle",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] + add_field("qcld",scalar3d_layout_mid , n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] + add_field("ptend_q", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne aerosols [#/kg] + add_field("tendnd",scalar3d_layout_mid , n_unit/s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] + add_field("factnum", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam_coupling::num_aero_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] + add_field("ndropcol",scalar3d_layout_mid , n_unit/s, grid_name); // + add_field("ndropmix",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio tendency due to mixing [#/kg/s] + add_field("nsource",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] + add_field("wtke",scalar3d_layout_mid , n_unit/s, grid_name); // + add_field("ccn", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, grid_name); //number conc of aerosols activated at supersat [#/m^3] + // note: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud [#/m^2/s]] + // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] + add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output + add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency + + + //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + //interstitial aerosol tracers of interest: number (n) mixing ratios + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + //cloudborne aerosol tracers of interest: number (n) mixing ratios + //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); + + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); + if (strlen(cld_mmr_field_name) > 0) + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); + } + } + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + } + + //Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM + //These outputs should come from the cloud macrophysics process (e.g., SHOC) + auto m2 = m*m; + m2.set_string("m^2"); + auto s2 = s*s; + s2.set_string("s^2"); + + //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time + add_field("w_sec", scalar3d_layout_int, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints + + add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints + add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints + add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat + add_field("zm", scalar3d_layout_int, m, grid_name); // Eddy diffusivity for heat + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + const int num_aero_modes = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes} }; // mid points + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols + + + auto cm = m/100; + auto frz_unit = 1/(cm*cm*cm*s); // units of number mixing ratios of tracers + n_unit.set_string("1(cm^-3 s^-1)"); + + add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + // + /* + * NOTE on other inputs for the aci process: + * 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density + * 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using + * pseudo_density, p_mid, T_mid and qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). + * qv_mid can be obtained from "get_field_in" call + */ + +} + +void MAMAci::initialize_impl(const RunType run_type) { + m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); + + // set atmosphere state data + pdel_ = get_field_in("pseudo_density").get_view(); + p_int_ = get_field_in("p_int").get_view(); + w_sec_ = get_field_in("w_sec").get_view(); + state_q_ = get_field_in("state_q").get_view(); + ncldwtr_ = get_field_in("ncldwtr").get_view(); + qqcw_input_ = get_field_in("qqcw").get_view(); + dgnum_ = get_field_in("dgnum").get_view(); + nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); + niim_ = get_field_out("icenuc_num_immfrz").get_view(); + nidep_ = get_field_out("icenuc_num_depnuc").get_view(); + nimey_ = get_field_out("icenuc_num_meydep").get_view(); + naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); + naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); + qcld_ = get_field_out("qcld").get_view(); + ptend_q_output_ = get_field_out("ptend_q").get_view(); + tendnd_ = get_field_out("tendnd").get_view(); + factnum_ = get_field_out("factnum").get_view(); + ndropcol_ = get_field_out("ndropcol").get_view(); + ndropmix_ = get_field_out("ndropmix").get_view(); + nsource_ = get_field_out("nsource").get_view(); + wtke_ = get_field_out("wtke").get_view(); + ccn_ = get_field_out("ccn").get_view(); + coltend_outp_ = get_field_out("coltend").get_view(); + coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + kvh_ = get_field_in("kvh").get_view(); + zm_ = get_field_in("zm").get_view(); + + wet_atmosphere_.qc = get_field_in("qc").get_view(); + wet_atmosphere_.nc = get_field_in("nc").get_view(); + wet_atmosphere_.qi = get_field_in("qi").get_view(); + wet_atmosphere_.omega = get_field_in("omega").get_view(); + + dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); + dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); + dry_atmosphere_.qv = get_field_out("qv").get_view(); + dry_atmosphere_.z_mid = get_field_out("z_mid").get_view(); + dry_atmosphere_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); + + hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); + hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); + hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + + stratiform_cloud_fraction_ = get_field_out("stratiform_cloud_fraction").get_view(); + activation_fraction_accum_idx_ = get_field_out("activation_fraction_accum").get_view(); + activation_fraction_coarse_idx_ = get_field_out("activation_fraction_coarse").get_view(); + + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + auto prog_index = [](const int mode, const int species) { + const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); + const int ind = + (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; + return ind; + }; + //interstitial aerosol tracers of interest: number (n) mixing ratios + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + aerosol_state_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + + //cloudborne aerosol tracers of interest: number (n) mixing ratios + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + aerosol_state_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); + + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) { + const int index = prog_index(m, a); + aerosol_state_.int_aero_mmr[m][index] = get_field_out(int_mmr_field_name).get_view(); + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); + if (strlen(cld_mmr_field_name) > 0) { + const int index = prog_index(m, a); + aerosol_state_.cld_aero_mmr[m][index] = get_field_out(cld_mmr_field_name).get_view(); + } + } + } + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + aerosol_state_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + } + + for (int i=0; ilog(ekat::logger::LogLevel::info,"calling ACI run"); + + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + + copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_, mam4::ndrop::ncnst_tot); + + // All the inputs are available to compute w0 and rho + // Convert from omega to w (vertical velocity) + // Negative omega means rising motion + compute_w0_and_rho(team_policy, w0_, rho_, wet_atmosphere_, dry_atmosphere_, top_lev_, nlev_); + + compute_tke_using_w_sec(team_policy, tke_, w_sec_, nlev_); + Kokkos::fence(); // wait for for tke_ to be computed. + + compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, wsubmin, top_lev_, nlev_); + Kokkos::fence(); // wait for wsig_ to be computed. + + compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); + + compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); + Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + + compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, + nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, + aerosol_state_, dry_atmosphere_, aitken_dry_dia_, nlev_); + + store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, wet_atmosphere_, liqcldf_, top_lev_); + + //------------------------------------------------------------- + // Save cloud borne aerosols to be used in the heterozenous + // freezing before they are changed by the droplet activation + // process. This is only a select subset of cloud borne + // aerosols, not all the cloud borne aerosols. + //------------------------------------------------------------- + /*NOTE: We probably need to store indices for the select few cloud borne aerosols + Fortran code: + lchnk_zb = lchnk - begchunk + ! save copy of cloud borne aerosols for use in heterogeneous freezing before + ! we change it in dropmixnuc + do ispec = 1, ncnst + call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) + aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) + aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * rho(:ncol,:) + enddo + */ + + compute_recipical_pseudo_density(team_policy, rpdel_, pdel_, nlev_); + Kokkos::fence(); // wait for rpdel_ to be computed. + + call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, + raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, + p_int_, pdel_, rpdel_, zm_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + Kokkos::fence(); // wait for ptend_q_ to be computed. + + copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, mam4::ndrop::nvar_ptend_q, nlev_); + copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, mam4::ndrop::ncnst_tot, nlev_); + copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, mam4::ndrop::ncnst_tot, nlev_); + + call_hetfrz_compute_tendencies(team_policy, + hetfrz_, aerosol_state_, wet_atmosphere_, dry_atmosphere_, + stratiform_cloud_fraction_, + activation_fraction_accum_idx_, activation_fraction_coarse_idx_, + hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, + naai_hom_, naai_, + diagnostic_scratch_, + nlev_); + + Kokkos::fence(); // wait before returning to calling function } void MAMAci::finalize_impl(){ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index c5ed6a5a90c..046d8578112 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -23,11 +23,12 @@ class MAMAci final : public scream::AtmosphereProcess { mam4::Hetfrz hetfrz_; // views for single- and multi-column data - using view_1d = typename KT::template view_1d; - using const_view_2d = typename KT::template view_2d; - using view_2d = typename KT::template view_2d; - using view_3d = typename KT::template view_3d; - using const_view_3d = typename KT::template view_3d; + using view_1d = scream::mam_coupling::view_1d; + using view_2d = scream::mam_coupling::view_2d; + using view_3d = scream::mam_coupling::view_3d; + using const_view_1d = scream::mam_coupling::const_view_1d; + using const_view_2d = scream::mam_coupling::const_view_2d; + using const_view_3d = scream::mam_coupling::const_view_3d; template using view_4d = KT::view; @@ -45,48 +46,15 @@ class MAMAci final : public scream::AtmosphereProcess { // turbulent kinetic energy [m^2/s^2] view_2d tke_; - const_view_2d qv_dry_; - const_view_2d cldfrac_; - const_view_2d w_updraft_; - view_2d aitken_dry_dia_; - view_2d qc_coarse_bc_; - view_2d qc_coarse_dst_; - view_2d qc_coarse_mom_; - view_2d qc_coarse_nacl_; - view_2d qc_coarse_pom_; - view_2d qc_coarse_soa_; - - view_2d qc_accum_bc_; - view_2d qc_accum_dst_; - view_2d qc_accum_mom_; - view_2d qc_accum_nacl_; - view_2d qc_accum_pom_; - view_2d qc_accum_so4_; - view_2d qc_accum_soa_; - - view_2d qi_accum_dst_; - view_2d qi_accum_so4_; - view_2d qi_accum_mom_; - view_2d qi_accum_bc_; - view_2d qi_accum_pom_; - view_2d qi_accum_soa_; - - view_2d qi_coarse_dst_; - view_2d qi_coarse_nacl_; - view_2d qi_coarse_so4_; - view_2d qi_coarse_mom_; - view_2d qi_coarse_bc_; - view_2d qi_coarse_pom_; - view_2d qi_coarse_soa_; - - view_2d qi_pcarbon_bc_; - view_2d qi_pcarbon_mom_; - view_2d qi_pcarbon_pom_; - view_2d nc_accum_; - view_2d ni_accum_; - view_2d ni_coarse_; - view_2d ni_aitken_; + + view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()][mam_coupling::num_aero_species()]; + mam_coupling::AerosolState aerosol_state_; + + mam_coupling::WetAtmosphere wet_atmosphere_; + + mam_coupling::DryAtmosphere dry_atmosphere_; + const_view_3d dgnum_; view_2d nihf_; view_2d niim_; @@ -95,20 +63,17 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d naai_hom_; view_2d naai_; const_view_2d liqcldf_; - const_view_2d qc_; - const_view_2d qi_; - const_view_2d nc_; const_view_2d kvh_; - view_2d cldo_; - view_2d zm_; - view_3d state_q_; - view_2d ncldwtr_; + const_view_2d zm_; + const_view_3d state_q_; + const_view_2d ncldwtr_; - view_2d lcldn_; - view_2d lcldo_; + view_2d cloud_frac_new_; + view_2d cloud_frac_old_; view_2d qcld_; view_2d tendnd_; + // ptend_q_ is just ptend_q_output_ reformatted. view_2d ptend_q_[mam4::ndrop::nvar_ptend_q]; view_3d ptend_q_output_; view_3d factnum_; @@ -123,8 +88,11 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d coltend_[mam4::ndrop::ncnst_tot]; view_3d coltend_cw_outp_; view_2d coltend_cw_[mam4::ndrop::ncnst_tot]; + + // raercol_cw_ and raercol_ are work arrays for dropmixnuc, allocated on the stack. view_2d raercol_cw_[mam4::ndrop::pver][2]; view_2d raercol_[mam4::ndrop::pver][2]; + view_3d nact_; view_3d mact_; view_2d dropmixnuc_scratch_mem_[15]; @@ -133,6 +101,8 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d activation_fraction_accum_idx_; view_2d activation_fraction_coarse_idx_; + // These are the output tendencies from heterogeneous freezing that need to be + // added correctly to the cloud-micorphysics scheme. view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; view_2d hetfrz_depostion_nucleation_tend_; @@ -144,6 +114,12 @@ class MAMAci final : public scream::AtmosphereProcess { // FIXME: This should be read in to make user selectable. const int top_lev_ = 6; + // local atmospheric state column variables + const_view_2d p_int_; // Total pressure [Pa] at interfaces + const_view_2d pdel_; // pressure thickess of layer [Pa] + view_2d rpdel_; // Inverse of pdel_ + const_view_2d w_sec_; // Vertical velocity variance + public: // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); @@ -165,23 +141,13 @@ class MAMAci final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; - // number of aerosol modes - int num_aero_modes_; - - // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. struct Preprocess { Preprocess() = default; - - - //const_view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] - // assigns local variables - void set_variables(const const_view_2d& pdel) { - //p1del_ = pdel; + void set_variables(const const_view_2d &pdel) { } // set_variables }; // MAMAci::Preprocess @@ -189,15 +155,6 @@ class MAMAci final : public scream::AtmosphereProcess { // pre- and postprocessing scratch pads Preprocess preprocess_; - // local atmospheric state column variables - const_view_2d omega_; // Vertical pressure velocity [Pa/s] at midpoints - const_view_2d p_mid_; // Total pressure [Pa] at midpoints - const_view_2d p_int_; // Total pressure [Pa] at interfaces - const_view_2d T_mid_; // Temperature[K] at midpoints - const_view_2d pdel_; // pressure thickess of layer [Pa] - view_2d rpdel_; // Inverse of pdel_ - const_view_2d w_sec_; // Vertical velocity variance - // physics grid for column information std::shared_ptr grid_; }; // MAMAci diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 278e97c033e..d0463388568 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -21,6 +21,7 @@ using view_2d = typename KT::template view_2d; using view_3d = typename KT::template view_3d; using const_view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; +using const_view_3d = typename KT::template view_3d; using complex_view_3d = typename KT::template view_3d>; using complex_view_2d = typename KT::template view_2d>; diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index a7103ad01a8..c87894362ae 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -4,9 +4,9 @@ driver_options: atmosphere_dag_verbosity_level: 5 time_stepping: - time_step: ${ATM_TIME_STEP} - run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} + time_step: 1 + run_t0: 2021-10-12-45000 # YYYY-MM-DD-XXXXX + number_of_steps: 24 atmosphere_processes: atm_procs_list: [mam4_aci] @@ -23,12 +23,29 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + Filename: /Users/jroverf/Codes/scream/build/atm/scream/init/screami_unit_tests_ne2np4L72_20220822.nc T_mid: 273.0 p_mid: 1.e5 num_a1: 1.e5 num_c1: 1.e5 bc_a1: 1.e-5 + qqcw: 1e5 + + aero_gas_mmr_o3: 0.0 + aero_gas_mmr_h2o2: 0.0 + aero_gas_mmr_h2so4: 0.0 + aero_gas_mmr_so2: 0.0 + aero_gas_mmr_dms: 0.0 + aero_gas_mmr_soag: 0.0 + + hetfrz_immersion_nucleation_tend: 0 + hetfrz_contact_nucleation_tend: 0 + hetfrz_depostion_nucleation_tend: 0 + + zm: 1e1 + + w_updraft: 1e0 + ptend: 1e0 num_a2: 1e5 num_a3: 1e5 num_a4: 1e5 @@ -56,6 +73,27 @@ initial_conditions: bc_a4: 1e-5 mom_a4: 1e-5 #cloud borne aerosols + qi_coarse_dst: 1e-5 + qi_coarse_nacl: 1e-5 + qi_coarse_so4: 1e-5 + qi_coarse_mom: 1e-5 + qi_coarse_bc: 1e-5 + qi_coarse_pom: 1e-5 + qi_coarse_soa: 1e-5 + ni_accum: 1e-5 + ni_coarse: 1e-5 + ni_aitken: 1e-5 + + state_q: 1e-5 + ncldwtr: 1e-12 + + icenuc_num_hetfrz: 1e3 + icenuc_num_immfrz: 1e3 + icenuc_num_depnuc: 1e3 + icenuc_num_meydep: 1e3 + num_act_aerosol_ice_nucle_hom: 1e3 + num_act_aerosol_ice_nucle: 1e3 + soa_c1: 1e-5 so4_c1: 1e-5 pom_c1: 1e-5 From 1b174de79338688b5cc3726abb2541a777334d38 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 25 Jan 2024 15:51:22 -0700 Subject: [PATCH 020/142] Fix accidental push of wrong file. --- components/eamxx/tests/uncoupled/mam4/input_aci.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index c87894362ae..40d3ebd9a98 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -4,9 +4,9 @@ driver_options: atmosphere_dag_verbosity_level: 5 time_stepping: - time_step: 1 - run_t0: 2021-10-12-45000 # YYYY-MM-DD-XXXXX - number_of_steps: 24 + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} atmosphere_processes: atm_procs_list: [mam4_aci] @@ -23,7 +23,7 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /Users/jroverf/Codes/scream/build/atm/scream/init/screami_unit_tests_ne2np4L72_20220822.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} T_mid: 273.0 p_mid: 1.e5 num_a1: 1.e5 @@ -46,6 +46,7 @@ initial_conditions: w_updraft: 1e0 ptend: 1e0 + num_a2: 1e5 num_a3: 1e5 num_a4: 1e5 From 7621745568cc717f860a585224d2ce7c7f51dfe1 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 25 Jan 2024 15:54:43 -0700 Subject: [PATCH 021/142] Fix accidental push of wrong file. --- components/eamxx/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index f0afd1c3045..8567512abf2 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -546,7 +546,7 @@ if (SCREAM_DOUBLE_PRECISION) set(SCREAM_Fortran_FLAGS -real-size 64) elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") #find what is crayclang id in cmake - set(SCREAM_Fortran_FLAGS -eZ) + set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() From 4880e6cdbe3e78c5f94d4a5521330f3fdc196496 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 31 Jan 2024 14:35:18 -0700 Subject: [PATCH 022/142] Fixes for Cuda compiler. Requires latest mam4xx to work. --- .../mam/eamxx_mam_aci_process_interface.cpp | 249 +++++++++++++----- 1 file changed, 181 insertions(+), 68 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 91de9d4a03a..e6be4a9c466 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -8,6 +8,22 @@ namespace scream namespace { +KOKKOS_INLINE_FUNCTION +void copy_scream_array_to_mam4xx( + const haero::ThreadTeam &team, + MAMAci::view_2d mam4xx_view[], + MAMAci::const_view_3d scream_view, + const int icol, + const int nlev, + const int num_views) +{ + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + for (int i=0; i; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol,kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol,kk) = p_mid(icol,kk)/(rair*T_mid(icol,kk)); + w0(icol,kk) = -1.0*omega(icol,kk)/(rho(icol,kk)*gravit); }); } void compute_w0_and_rho( @@ -35,21 +73,28 @@ void compute_w0_and_rho( { //Get physical constants using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] MAMAci::const_view_2d omega = wet_atmosphere.omega; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - w0(icol,kk) = 0; - rho(icol, kk) = -999.0; - }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - rho(icol,kk) = p_mid(icol,kk)/(rair*T_mid(icol,kk)); - w0(icol,kk) = -1.0*omega(icol,kk)/(rho(icol,kk)*gravit); - }); + compute_w0_and_rho( team, w0, rho, omega, T_mid, p_mid, icol, top_lev, nlev); + }); +} + +KOKKOS_INLINE_FUNCTION +void compute_tke_using_w_sec( + const haero::ThreadTeam &team, + MAMAci::view_2d tke, + MAMAci::const_view_2d w_sec, + const int icol, + const int nlev) +{ + // FIXME Is this the correct boundary condition for tke at the surface? + // TKE seems to be at interfaces but w_sec is at cell centers so this + // descrepensy needs to be worked out. + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { + tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); }); } void compute_tke_using_w_sec( @@ -60,39 +105,48 @@ void compute_tke_using_w_sec( { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // FIXME Is this the correct boundary condition for tke at the surface? - // TKE seems to be at interfaces but w_sec is at cell centers so this - // descrepensy needs to be worked out. - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { - tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); - }); + compute_tke_using_w_sec(team, tke, w_sec, icol, nlev); }); } +KOKKOS_INLINE_FUNCTION void compute_subgrid_scale_velocities( - haero::ThreadTeamPolicy team_policy, + const haero::ThreadTeam &team, MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, const Real wsubmin, + const int icol, const int top_lev, const int nlev) { // More refined computation of sub-grid vertical velocity // Set to be zero at the surface by initialization. + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = wsubmin; + wsubice(icol,kk) = 0.001; + wsig(icol,kk) = 0.001; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = haero::sqrt(0.5*(tke(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); + wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); + wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); + wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); + }); +} +void compute_subgrid_scale_velocities( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d wsub, + MAMAci::view_2d wsubice, + MAMAci::view_2d wsig, + MAMAci::const_view_2d tke, + const Real wsubmin, + const int top_lev, + const int nlev) +{ Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - wsub(icol,kk) = wsubmin; - wsubice(icol,kk) = 0.001; - wsig(icol,kk) = 0.001; - }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - wsub(icol,kk) = haero::sqrt(0.5*(tke(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); - wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); - wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); - wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); - }); + compute_subgrid_scale_velocities(team, wsub, wsubice, wsig, tke, wsubmin, icol, top_lev, nlev); }); } @@ -147,6 +201,19 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) } return ww; } +KOKKOS_INLINE_FUNCTION +void compute_subgrid_mean_updraft_velocities( + const haero::ThreadTeam &team, + MAMAci::view_2d w2, + MAMAci::const_view_2d w0, + MAMAci::const_view_2d wsig, + const int icol, + const int nlev) +{ + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); + }); +} void compute_subgrid_mean_updraft_velocities( haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w2, @@ -156,9 +223,20 @@ void compute_subgrid_mean_updraft_velocities( { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); - }); + compute_subgrid_mean_updraft_velocities(team, w2, w0, wsig, icol, nlev); + }); +} +KOKKOS_INLINE_FUNCTION +void compute_aitken_dry_diameter( + const haero::ThreadTeam &team, + MAMAci::view_2d aitken_dry_dia, + MAMAci::const_view_3d dgnum, + const int icol, + const int top_lev) +{ + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); }); } void compute_aitken_dry_diameter( @@ -169,10 +247,7 @@ void compute_aitken_dry_diameter( { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); - }); + compute_aitken_dry_diameter(team, aitken_dry_dia, dgnum, icol, top_lev); }); } @@ -252,12 +327,15 @@ void compute_nucleate_ice_tendencies( nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); }); } +KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( - haero::ThreadTeamPolicy team_policy, + const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, - mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::const_view_2d qc, + MAMAci::const_view_2d qi, MAMAci::const_view_2d liqcldf, + const int icol, const int top_lev) { //------------------------------------------------------------- @@ -270,21 +348,43 @@ void store_liquid_cloud_fraction( // a note of it and use the new cloud fraction for the // old cloud fraction. //------------------------------------------------------------- + static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + const Real qcld = qc(icol,kk) + qi(icol,kk); + if (qcld > qsmall) { + cloud_frac_new(icol,kk)=liqcldf(icol,kk); + cloud_frac_old(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old + } else { + cloud_frac_new(icol,kk)=0; + cloud_frac_old(icol,kk)=0; + } + }); +} +void store_liquid_cloud_fraction( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::const_view_2d liqcldf, + const int top_lev) +{ MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d qi = wet_atmosphere.qi; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - const Real qcld = qc(icol,kk) + qi(icol,kk); - if (qcld > qsmall) { - cloud_frac_new(icol,kk)=liqcldf(icol,kk); - cloud_frac_old(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old - } else { - cloud_frac_new(icol,kk)=0; - cloud_frac_old(icol,kk)=0; - } - }); + store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, qi, liqcldf, icol, top_lev); + }); +} +KOKKOS_INLINE_FUNCTION +void compute_recipical_pseudo_density( + const haero::ThreadTeam &team, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d pdel, + const int icol, + const int nlev) +{ + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + rpdel(icol,kk) = 1/pdel(icol,kk); }); } void compute_recipical_pseudo_density( @@ -295,9 +395,7 @@ void compute_recipical_pseudo_density( { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - rpdel(icol,kk) = 1/pdel(icol,kk); - }); + compute_recipical_pseudo_density(team, rpdel, pdel, icol, nlev); }); } @@ -468,6 +566,21 @@ void call_function_dropmixnuc( ekat::subview(qqcwtend, icol) ); }); } +KOKKOS_INLINE_FUNCTION +void copy_mam4xx_array_to_scream( + const haero::ThreadTeam &team, + MAMAci::view_3d scream, + MAMAci::view_2d mam4xx[], + const int icol, + const int len, + const int nlev) +{ + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + for (int i=0; i grids_manager) FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces - using namespace ekat::units; + ekat::units::Units kg = ekat::units::kg; + ekat::units::Units Pa = ekat::units::Pa; + ekat::units::Units s = ekat::units::s; + ekat::units::Units m = ekat::units::m; + ekat::units::Units K = ekat::units::K; auto q_unit = kg/kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); auto n_unit = 1/kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); - auto nondim = Units::nondimensional(); + auto nondim = ekat::units::Units::nondimensional(); // atmospheric quantities add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] @@ -711,25 +824,25 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for (int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { //interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); //cloudborne aerosol tracers of interest: number (n) mixing ratios //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); if (strlen(int_mmr_field_name) > 0) add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); if (strlen(cld_mmr_field_name) > 0) add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } From 161b8ed4ca740669b8564a682e5306865838ffc9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 6 Feb 2024 12:07:21 -0800 Subject: [PATCH 023/142] Fixes broken aci test after rebase --- .../eamxx/src/physics/mam/mam_coupling.hpp | 3 +- .../tests/uncoupled/mam4/aci/CMakeLists.txt | 24 +++++++ .../mam4/{input_aci.yaml => aci/input.yaml} | 2 +- .../{mam4_aci_output.yaml => aci/output.yaml} | 0 .../uncoupled/mam4/mam4_aci_standalone.cpp | 68 ------------------- 5 files changed, 27 insertions(+), 70 deletions(-) create mode 100644 components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt rename components/eamxx/tests/uncoupled/mam4/{input_aci.yaml => aci/input.yaml} (98%) rename components/eamxx/tests/uncoupled/mam4/{mam4_aci_output.yaml => aci/output.yaml} (100%) delete mode 100644 components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index d0463388568..6436c0c80ff 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -606,7 +606,8 @@ void compute_updraft_velocities(const Team& team, constexpr int nlev = mam4::nlev; int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); + dry_atm.dz(i,k) = PF::calculate_dz(dry_atm.p_del(i,k), dry_atm.p_mid(i,k), dry_atm.T_mid(i,k), wet_atm.qv(i,k)); + const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); }); } diff --git a/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt new file mode 100644 index 00000000000..45532850822 --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt @@ -0,0 +1,24 @@ +include (ScreamUtils) + + + +# Create the test +CreateADUnitTest(mam4_aci_standalone + LABELS mam4 aci physics driver + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1) +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml similarity index 98% rename from components/eamxx/tests/uncoupled/mam4/input_aci.yaml rename to components/eamxx/tests/uncoupled/mam4/aci/input.yaml index 40d3ebd9a98..dc807241f96 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml @@ -124,5 +124,5 @@ initial_conditions: dgnum: 1e-3 # The parameters for I/O control Scorpio: - output_yaml_files: ["mam4_aci_output.yaml"] + output_yaml_files: ["output.yaml"] ... diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml b/components/eamxx/tests/uncoupled/mam4/aci/output.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml rename to components/eamxx/tests/uncoupled/mam4/aci/output.yaml diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp deleted file mode 100644 index dd3ed62ec15..00000000000 --- a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/register_physics.hpp" -#include "physics/mam/eamxx_mam_aci_process_interface.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" - -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/logging/ekat_logger.hpp" - -#include - -namespace scream { - -TEST_CASE("mam4-aci-standalone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - ekat::logger::Logger<> logger("mam4-aci", - ekat::logger::LogLevel::debug, atm_comm); - - // Load ad parameter list - std::string fname = "input_aci.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - logger.debug("aci yaml parsed."); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - logger.info("running MAMAci standalone test with dt = {} for {} steps.", dt, nsteps); - - // Need to register products in the factory *before* we create any atm process or grids manager. - register_physics(); - register_mesh_free_grids_manager(); - register_diagnostics(); - logger.debug("products registered."); - - // Create the driver - AtmosphereDriver ad; - logger.debug("driver created."); - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - logger.debug("driver initialized."); - - logger.info("Start time stepping loop ... [0%]"); - for (int i=0; i Date: Mon, 5 Feb 2024 14:39:49 -0700 Subject: [PATCH 024/142] Finally fix all the problems with running on the GPU. --- .../mam/eamxx_mam_aci_process_interface.cpp | 150 +++++++++++------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 - 2 files changed, 95 insertions(+), 56 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e6be4a9c466..b71052fb2d5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -11,29 +11,33 @@ namespace KOKKOS_INLINE_FUNCTION void copy_scream_array_to_mam4xx( const haero::ThreadTeam &team, - MAMAci::view_2d mam4xx_view[], + const MAMAci::view_2d mam4xx_view, MAMAci::const_view_3d scream_view, const int icol, const int nlev, - const int num_views) + const int view_num) { Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - for (int i=0; i void copy_mam4xx_array_to_scream( haero::ThreadTeamPolicy team_policy, MAMAci::view_3d scream, - MAMAci::view_2d mam4xx[], - const int len, + MAMAci::view_2d mam4xx[len], const int nlev) { + // Localize the input arrays. + MAMAci::view_2d mam4xx_loc[len]; + for (int view_num=0; view_num < len; ++view_num) + mam4xx_loc[view_num] = mam4xx[view_num]; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - copy_mam4xx_array_to_scream(team, scream, mam4xx, icol, len, nlev); + for (int view_num=0; view_num < len; ++view_num) { + MAMAci::view_2d mam4xx_view = mam4xx_loc[view_num]; + copy_mam4xx_array_to_scream(team, scream, mam4xx_view, icol, nlev, view_num); + } }); } void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, - mam4::Hetfrz &hetfrz, - mam_coupling::AerosolState &aerosol_state, - mam_coupling::WetAtmosphere &wet_atmosphere, - mam_coupling::DryAtmosphere &dry_atmosphere, + mam4::Hetfrz &hetfrz_, + mam_coupling::AerosolState &aerosol_state_, + mam_coupling::WetAtmosphere &wet_atmosphere_, + mam_coupling::DryAtmosphere &dry_atmosphere_, MAMAci::view_2d stratiform_cloud_fraction, MAMAci::view_2d activation_fraction_accum_idx, MAMAci::view_2d activation_fraction_coarse_idx, @@ -608,15 +640,25 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, - MAMAci::view_2d diagnostic_scratch[], + MAMAci::view_2d diagnostic_scratch_[], const int nlev) { using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); + + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState aerosol_state = aerosol_state_; + mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; + MAMAci::view_2d diagnostic_scratch[42]; + for (int i=0; i<42; ++i) + diagnostic_scratch[i] = diagnostic_scratch_[i];; + MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d nc = wet_atmosphere.nc; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -629,7 +671,7 @@ void call_hetfrz_compute_tendencies( dummy, ekat::subview(qc, icol), ekat::subview(nc, icol), - dummy, dummy, dummy, dummy, dummy, dummy, + dummy, dummy, dummy, dummy, dummy, dummy, dummy, pblh); // set surface state data haero::Surface surf{}; @@ -865,7 +907,6 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat - add_field("zm", scalar3d_layout_int, m, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -921,7 +962,6 @@ void MAMAci::initialize_impl(const RunType run_type) { coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); kvh_ = get_field_in("kvh").get_view(); - zm_ = get_field_in("zm").get_view(); wet_atmosphere_.qc = get_field_in("qc").get_view(); wet_atmosphere_.nc = get_field_in("nc").get_view(); @@ -1021,7 +1061,7 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_, mam4::ndrop::ncnst_tot); + copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_); // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) @@ -1068,14 +1108,14 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, - p_int_, pdel_, rpdel_, zm_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + p_int_, pdel_, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. - copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, mam4::ndrop::nvar_ptend_q, nlev_); - copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, mam4::ndrop::ncnst_tot, nlev_); - copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, mam4::ndrop::ncnst_tot, nlev_); + copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, nlev_); + copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, nlev_); + copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, nlev_); call_hetfrz_compute_tendencies(team_policy, hetfrz_, aerosol_state_, wet_atmosphere_, dry_atmosphere_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 046d8578112..2c590791744 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -65,7 +65,6 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - const_view_2d zm_; const_view_3d state_q_; const_view_2d ncldwtr_; From 608369a6f69e0e2b895f83ce7ef3a74170016eb2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 7 Feb 2024 08:41:55 -0800 Subject: [PATCH 025/142] Fixes compilation errors and update submodules of mam4xx and haero --- .../eamxx/src/physics/mam/mam_coupling.hpp | 87 ++++++++++--------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 6436c0c80ff..89966b591a8 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -23,9 +23,6 @@ using const_view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; using const_view_3d = typename KT::template view_3d; -using complex_view_3d = typename KT::template view_3d>; -using complex_view_2d = typename KT::template view_2d>; - // Kokkos thread team (league member) using Team = Kokkos::TeamPolicy::member_type; @@ -35,8 +32,6 @@ using uview_2d = typename ekat::template Unmanaged; -using view_int_1d = typename KT::template view_1d; - // number of constituents in gas chemistry "work arrays" KOKKOS_INLINE_FUNCTION constexpr int gas_pcnst() { @@ -77,7 +72,7 @@ constexpr int num_aero_tracers() { // Given a MAM aerosol mode index, returns a string denoting the symbolic // name of the mode. -inline +KOKKOS_INLINE_FUNCTION const char* aero_mode_name(const int mode) { static const char *mode_names[num_aero_modes()] = { "1", @@ -88,17 +83,33 @@ const char* aero_mode_name(const int mode) { return mode_names[mode]; } +// Given a MAM aerosol species ID, returns a string denoting the symbolic +// name of the species. +KOKKOS_INLINE_FUNCTION +const char* aero_species_name(const int species_id) { + static const char *species_names[num_aero_species()] = { + "soa", + "so4", + "pom", + "bc", + "nacl", + "dst", + "mom", + }; + return species_names[species_id]; +} + // Given a MAM aerosol-related gas ID, returns a string denoting the symbolic // name of the gas species. -inline +KOKKOS_INLINE_FUNCTION const char* gas_species_name(const int gas_id) { static const char *species_names[num_aero_gases()] = { - "O3", - "H2O2", - "H2SO4", - "SO2", - "DMS", - "SOAG" + "o3", + "h2o2", + "h2so4", + "so2", + "dms", + "soag" }; return species_names[gas_id]; } @@ -111,14 +122,14 @@ constexpr int max_field_name_len() { return 128; } -inline +KOKKOS_INLINE_FUNCTION size_t gpu_strlen(const char* s) { size_t l = 0; while (s[l]) ++l; return l; } -inline +KOKKOS_INLINE_FUNCTION void concat_2_strings(const char *s1, const char *s2, char *concatted) { size_t len1 = gpu_strlen(s1); for (size_t i = 0; i < len1; ++i) @@ -129,7 +140,7 @@ void concat_2_strings(const char *s1, const char *s2, char *concatted) { concatted[len1+len2] = 0; } -inline +KOKKOS_INLINE_FUNCTION void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { size_t len1 = gpu_strlen(s1); for (size_t i = 0; i < len1; ++i) @@ -143,31 +154,31 @@ void concat_3_strings(const char *s1, const char *s2, const char *s3, char *conc concatted[len1+len2+len3] = 0; } -inline +KOKKOS_INLINE_FUNCTION char* int_aero_nmr_names(int mode) { static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return int_aero_nmr_names_[mode]; } -inline +KOKKOS_INLINE_FUNCTION char* cld_aero_nmr_names(int mode) { static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return cld_aero_nmr_names_[mode]; } -inline +KOKKOS_INLINE_FUNCTION char* int_aero_mmr_names(int mode, int species) { static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return int_aero_mmr_names_[mode][species]; } -inline +KOKKOS_INLINE_FUNCTION char* cld_aero_mmr_names(int mode, int species) { static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return cld_aero_mmr_names_[mode][species]; } -inline +KOKKOS_INLINE_FUNCTION char* gas_mmr_names(int gas_id) { static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; return gas_mmr_names_[gas_id]; @@ -177,7 +188,7 @@ char* gas_mmr_names(int gas_id) { // Given a MAM aerosol mode index, returns the name of the related interstitial // modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") -inline +KOKKOS_INLINE_FUNCTION const char* int_aero_nmr_field_name(const int mode) { if (!int_aero_nmr_names(mode)[0]) { concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); @@ -187,7 +198,7 @@ const char* int_aero_nmr_field_name(const int mode) { // Given a MAM aerosol mode index, returns the name of the related cloudborne // modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") -inline +KOKKOS_INLINE_FUNCTION const char* cld_aero_nmr_field_name(const int mode) { if (!cld_aero_nmr_names(mode)[0]) { concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); @@ -200,13 +211,12 @@ const char* cld_aero_nmr_field_name(const int mode) { // field in EAMxx. The form of the field name is "_a<1-based-mode-index>". // If the desired species is not present within the desire mode, returns a blank // string (""). -inline +KOKKOS_INLINE_FUNCTION const char* int_aero_mmr_field_name(const int mode, const int species) { if (!int_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); if (aero_id != mam4::AeroId::None) { - auto aerosol_species_name =mam4::aero_id_short_name(aero_id); - concat_3_strings(aerosol_species_name.c_str(), + concat_3_strings(aero_species_name(static_cast(aero_id)), "_a", aero_mode_name(mode), int_aero_mmr_names(mode, species)); } @@ -219,13 +229,12 @@ const char* int_aero_mmr_field_name(const int mode, const int species) { // field in EAMxx. The form of the field name is "_c<1-based-mode-index>". // If the desired species is not present within the desire mode, returns a blank // string (""). -inline +KOKKOS_INLINE_FUNCTION const char* cld_aero_mmr_field_name(const int mode, const int species) { if (!cld_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); if (aero_id != mam4::AeroId::None) { - auto aerosol_species_name =mam4::aero_id_short_name(aero_id); - concat_3_strings(aerosol_species_name.c_str(), + concat_3_strings(aero_species_name(static_cast(aero_id)), "_c", aero_mode_name(mode), cld_aero_mmr_names(mode, species)); } @@ -234,10 +243,13 @@ const char* cld_aero_mmr_field_name(const int mode, const int species) { }; // Given a MAM aerosol-related gas identifier, returns the name of its mass -// mixing ratio field in EAMxx -inline +// mixing ratio field in EAMxx ("aero_gas_mmr_") +KOKKOS_INLINE_FUNCTION const char* gas_mmr_field_name(const int gas) { - return const_cast(gas_species_name(gas)); + if (!gas_mmr_names(gas)[0]) { + concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); + } + return const_cast(gas_mmr_names(gas)); } // This type stores multi-column views related specifically to the wet @@ -579,19 +591,14 @@ void compute_vertical_layer_heights(const Team& team, "Given column index does not correspond to given team!"); const auto dz = ekat::subview(dry_atm.dz, column_index); - const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); - const auto qv = ekat::subview(dry_atm.qv, column_index); - const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); - const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); - const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); - PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, dz); - team.team_barrier(); + auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + auto z_mid = ekat::subview(dry_atm.z_mid, column_index); PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, z_iface); team.team_barrier(); // likely necessary to have z_iface up to date PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } + // Given a thread team and wet and dry atmospheres, dispatches threads from the // team to compute the vertical updraft velocity for the column with the given // index. From 26a1c50bbe093323119ba968b30030713279e103 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 18 Feb 2024 14:18:24 -0800 Subject: [PATCH 026/142] Add wet/dry atm and aerosol states with buffer --- .../mam/eamxx_mam_aci_process_interface.cpp | 63 +++++++++++++++---- .../mam/eamxx_mam_aci_process_interface.hpp | 51 ++++++++++++--- 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b71052fb2d5..7c545d1e752 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -629,7 +629,7 @@ void copy_mam4xx_array_to_scream( void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, - mam_coupling::AerosolState &aerosol_state_, + mam_coupling::AerosolState &dry_aero_, mam_coupling::WetAtmosphere &wet_atmosphere_, mam_coupling::DryAtmosphere &dry_atmosphere_, MAMAci::view_2d stratiform_cloud_fraction, @@ -647,7 +647,7 @@ void call_hetfrz_compute_tendencies( view_1d dummy("DummyView", nlev); mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState aerosol_state = aerosol_state_; + mam_coupling::AerosolState aerosol_state = dry_aero_; mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; MAMAci::view_2d diagnostic_scratch[42]; @@ -816,9 +816,11 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) n_unit.set_string("#/kg"); auto nondim = ekat::units::Units::nondimensional(); // atmospheric quantities + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // specific humidity [kg/kg] add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud ice number mixing ratio [1/kg] add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints @@ -932,6 +934,21 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) } +size_t MAMAci::requested_buffer_size_in_bytes() const +{ + return mam_coupling::buffer_size(ncol_, nlev_); +} + +void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMMicrophysics."); +} + + void MAMAci::initialize_impl(const RunType run_type) { m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); @@ -963,15 +980,20 @@ void MAMAci::initialize_impl(const RunType run_type) { liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); kvh_ = get_field_in("kvh").get_view(); + wet_atmosphere_.qv = get_field_in("qv").get_view(); wet_atmosphere_.qc = get_field_in("qc").get_view(); wet_atmosphere_.nc = get_field_in("nc").get_view(); wet_atmosphere_.qi = get_field_in("qi").get_view(); + wet_atmosphere_.ni = get_field_in("ni").get_view(); wet_atmosphere_.omega = get_field_in("omega").get_view(); dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); - dry_atmosphere_.qv = get_field_out("qv").get_view(); - dry_atmosphere_.z_mid = get_field_out("z_mid").get_view(); + dry_atmosphere_.qv = buffer_.qv_dry; + dry_atmosphere_.qc = buffer_.qc_dry; + dry_atmosphere_.nc = buffer_.nc_dry; + dry_atmosphere_.qi = buffer_.qi_dry; + dry_atmosphere_.ni = buffer_.ni_dry; dry_atmosphere_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); @@ -994,31 +1016,37 @@ void MAMAci::initialize_impl(const RunType run_type) { }; //interstitial aerosol tracers of interest: number (n) mixing ratios const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - aerosol_state_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; //cloudborne aerosol tracers of interest: number (n) mixing ratios const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - aerosol_state_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); + wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { const int index = prog_index(m, a); - aerosol_state_.int_aero_mmr[m][index] = get_field_out(int_mmr_field_name).get_view(); + wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); + std::cout<<"BALLI:-----"< 0) { const int index = prog_index(m, a); - aerosol_state_.cld_aero_mmr[m][index] = get_field_out(cld_mmr_field_name).get_view(); + wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - aerosol_state_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } for (int i=0; ilog(ekat::logger::LogLevel::info,"calling ACI run"); + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_); @@ -1081,7 +1119,7 @@ void MAMAci::run_impl(const double dt) { compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, - aerosol_state_, dry_atmosphere_, aitken_dry_dia_, nlev_); + dry_aero_, dry_atmosphere_, aitken_dry_dia_, nlev_); store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, wet_atmosphere_, liqcldf_, top_lev_); @@ -1105,7 +1143,7 @@ void MAMAci::run_impl(const double dt) { compute_recipical_pseudo_density(team_policy, rpdel_, pdel_, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. - +#if 0 call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, p_int_, pdel_, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, @@ -1118,7 +1156,7 @@ void MAMAci::run_impl(const double dt) { copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, nlev_); call_hetfrz_compute_tendencies(team_policy, - hetfrz_, aerosol_state_, wet_atmosphere_, dry_atmosphere_, + hetfrz_, dry_aero_, wet_atmosphere_, dry_atmosphere_, stratiform_cloud_fraction_, activation_fraction_accum_idx_, activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, @@ -1127,6 +1165,7 @@ void MAMAci::run_impl(const double dt) { nlev_); Kokkos::fence(); // wait before returning to calling function +#endif } void MAMAci::finalize_impl(){ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 2c590791744..336770092d8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -15,8 +15,6 @@ namespace scream class MAMAci final : public scream::AtmosphereProcess { -public: - using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; @@ -33,10 +31,10 @@ class MAMAci final : public scream::AtmosphereProcess { template using view_4d = KT::view; +//FIXME:B: Should the following variables be public? They are like that in micriphysics and optics codes // FIXME the time step for microphysics [s] need to get from the input const Real dtmicro_ = .0001; -private: // rho is air density [kg/m3] view_2d rho_; @@ -49,7 +47,6 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d aitken_dry_dia_; view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()][mam_coupling::num_aero_species()]; - mam_coupling::AerosolState aerosol_state_; mam_coupling::WetAtmosphere wet_atmosphere_; @@ -129,6 +126,10 @@ class MAMAci final : public scream::AtmosphereProcess { // grid void set_grids(const std::shared_ptr grids_manager) override; + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; @@ -145,15 +146,51 @@ class MAMAci final : public scream::AtmosphereProcess { // This functor implements this step, which is called during run_impl. struct Preprocess { Preprocess() = default; - // assigns local variables - void set_variables(const const_view_2d &pdel) { - } // set_variables + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere& wet_atm, + const mam_coupling::AerosolState& wet_aero, + const mam_coupling::DryAtmosphere& dry_atm, + const mam_coupling::AerosolState& dry_aero) { + + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); + team.team_barrier(); + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess // pre- and postprocessing scratch pads Preprocess preprocess_; + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // workspace manager for internal local variables + mam_coupling::Buffer buffer_; + // physics grid for column information std::shared_ptr grid_; }; // MAMAci From 9da7f7092b4bc57b710af8ceb137ec8e919f677e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 19 Feb 2024 18:19:57 -0800 Subject: [PATCH 027/142] Replaces pmid and zmid with quanties coming from EAMxx --- .../mam/eamxx_mam_aci_process_interface.cpp | 13 ++++++--- .../mam/eamxx_mam_aci_process_interface.hpp | 3 +++ .../eamxx/src/physics/mam/mam_coupling.hpp | 27 +++++++++++++++++-- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7c545d1e752..f3e8346b832 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -833,7 +833,6 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field ("z_mid", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] @@ -953,7 +952,7 @@ void MAMAci::initialize_impl(const RunType run_type) { m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); // set atmosphere state data - pdel_ = get_field_in("pseudo_density").get_view(); + p_int_ = get_field_in("p_int").get_view(); w_sec_ = get_field_in("w_sec").get_view(); state_q_ = get_field_in("state_q").get_view(); @@ -989,11 +988,17 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); + dry_atmosphere_.p_del = get_field_in("pseudo_density").get_view(); dry_atmosphere_.qv = buffer_.qv_dry; dry_atmosphere_.qc = buffer_.qc_dry; dry_atmosphere_.nc = buffer_.nc_dry; dry_atmosphere_.qi = buffer_.qi_dry; dry_atmosphere_.ni = buffer_.ni_dry; + + dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) + dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above surface at interface levels (m) + dry_atmosphere_.z_mid = buffer_.z_mid; // geopotential height above surface at mid levels (m) + dry_atmosphere_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); @@ -1141,12 +1146,12 @@ void MAMAci::run_impl(const double dt) { enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, pdel_, nlev_); + compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. #if 0 call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, - p_int_, pdel_, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + p_int_, dry_atmosphere_.p_del, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 336770092d8..224314c5099 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -169,6 +169,9 @@ class MAMAci final : public scream::AtmosphereProcess { compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); + //vertical heights has to be computed after computing dry mixing ratios for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + } // operator() // local variables for preprocess struct diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 89966b591a8..c864a727550 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -583,7 +583,7 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, // Given a thread team and a dry atmosphere state, dispatches threads from the // team to compute vertical layer heights and interfaces for the column with // the given index. -KOKKOS_INLINE_FUNCTION +/*KOKKOS_INLINE_FUNCTION void compute_vertical_layer_heights(const Team& team, const DryAtmosphere& dry_atm, const int column_index) { @@ -596,8 +596,31 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, z_iface); team.team_barrier(); // likely necessary to have z_iface up to date PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); -} +}*/ +KOKKOS_INLINE_FUNCTION +void compute_vertical_layer_heights(const Team& team, + const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + const auto dz = ekat::subview(dry_atm.dz, column_index); + const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine + const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); + const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); + + PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs + dz);//output + team.team_barrier(); + PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs + z_iface); //output + team.team_barrier(); // likely necessary to have z_iface up to date + PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); +} // Given a thread team and wet and dry atmospheres, dispatches threads from the // team to compute the vertical updraft velocity for the column with the given From 8cf3ad79a5aaa2a666dc78a5269245074975cd42 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 13 Mar 2024 20:56:20 -0700 Subject: [PATCH 028/142] clag format+ some minor movement of variables --- .../mam/eamxx_mam_aci_process_interface.cpp | 1823 +++++++++-------- .../mam/eamxx_mam_aci_process_interface.hpp | 100 +- 2 files changed, 980 insertions(+), 943 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index f3e8346b832..4b322245c3f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,185 +1,163 @@ #include + #include "ekat/util/ekat_units.hpp" #include "mam4xx/aero_config.hpp" #include "mam4xx/ndrop.hpp" -namespace scream -{ +namespace scream { -namespace -{ +namespace { KOKKOS_INLINE_FUNCTION -void copy_scream_array_to_mam4xx( - const haero::ThreadTeam &team, - const MAMAci::view_2d mam4xx_view, - MAMAci::const_view_3d scream_view, - const int icol, - const int nlev, - const int view_num) -{ - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - mam4xx_view(icol,kk) = scream_view(icol, kk, view_num); - }); +void copy_scream_array_to_mam4xx(const haero::ThreadTeam &team, + const MAMAci::view_2d mam4xx_view, + MAMAci::const_view_3d scream_view, + const int icol, const int nlev, + const int view_num) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + mam4xx_view(icol, kk) = scream_view(icol, kk, view_num); + }); } void copy_scream_array_to_mam4xx( haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d mam4xx_views[mam4::ndrop::ncnst_tot], - MAMAci::const_view_3d scream_view, - const int nlev) -{ + MAMAci::view_2d mam4xx_views[mam4::ndrop::ncnst_tot], + MAMAci::const_view_3d scream_view, const int nlev) { // Localize the input arrays. MAMAci::view_2d mam4xx[mam4::ndrop::ncnst_tot]; - for (int view_num=0; view_num; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - w0(icol,kk) = 0; - rho(icol, kk) = -999.0; - }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - rho(icol,kk) = p_mid(icol,kk)/(rair*T_mid(icol,kk)); - w0(icol,kk) = -1.0*omega(icol,kk)/(rho(icol,kk)*gravit); - }); +void compute_w0_and_rho(const haero::ThreadTeam &team, MAMAci::view_2d w0, + MAMAci::view_2d rho, MAMAci::const_view_2d omega, + MAMAci::const_view_2d T_mid, + MAMAci::const_view_2d p_mid, const int icol, + const int top_lev, const int nlev) { + // Get physical constants + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr auto rair = + C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol, kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); + w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); + }); } -void compute_w0_and_rho( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d w0, - MAMAci::view_2d rho, - mam_coupling::WetAtmosphere &wet_atmosphere, - mam_coupling::DryAtmosphere &dry_atmosphere, - const int top_lev, - const int nlev) -{ - //Get physical constants - using C = physics::Constants; +void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, + MAMAci::view_2d rho, + mam_coupling::WetAtmosphere &wet_atmosphere, + mam_coupling::DryAtmosphere &dry_atmosphere, + const int top_lev, const int nlev) { + // Get physical constants + using C = physics::Constants; MAMAci::const_view_2d omega = wet_atmosphere.omega; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_w0_and_rho( team, w0, rho, omega, T_mid, p_mid, icol, top_lev, nlev); - }); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_w0_and_rho(team, w0, rho, omega, T_mid, p_mid, icol, top_lev, + nlev); + }); } KOKKOS_INLINE_FUNCTION -void compute_tke_using_w_sec( - const haero::ThreadTeam &team, - MAMAci::view_2d tke, - MAMAci::const_view_2d w_sec, - const int icol, - const int nlev) -{ +void compute_tke_using_w_sec(const haero::ThreadTeam &team, MAMAci::view_2d tke, + MAMAci::const_view_2d w_sec, const int icol, + const int nlev) { // FIXME Is this the correct boundary condition for tke at the surface? // TKE seems to be at interfaces but w_sec is at cell centers so this // descrepensy needs to be worked out. - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { - tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); - }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev + 1), + KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); } -void compute_tke_using_w_sec( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d tke, - MAMAci::const_view_2d w_sec, - const int nlev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_tke_using_w_sec(team, tke, w_sec, icol, nlev); - }); +void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d tke, MAMAci::const_view_2d w_sec, + const int nlev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_tke_using_w_sec(team, tke, w_sec, icol, nlev); + }); } KOKKOS_INLINE_FUNCTION void compute_subgrid_scale_velocities( - const haero::ThreadTeam &team, - MAMAci::view_2d wsub, - MAMAci::view_2d wsubice, - MAMAci::view_2d wsig, - MAMAci::const_view_2d tke, - const Real wsubmin, - const int icol, - const int top_lev, - const int nlev) -{ + const haero::ThreadTeam &team, MAMAci::view_2d wsub, + MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, + const Real wsubmin, const int icol, const int top_lev, const int nlev) { // More refined computation of sub-grid vertical velocity // Set to be zero at the surface by initialization. - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - wsub(icol,kk) = wsubmin; - wsubice(icol,kk) = 0.001; - wsig(icol,kk) = 0.001; - }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - wsub(icol,kk) = haero::sqrt(0.5*(tke(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); - wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); - wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); - wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); - }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol, kk) = wsubmin; + wsubice(icol, kk) = 0.001; + wsig(icol, kk) = 0.001; + }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * + (2.0 / 3.0)); + wsig(icol, kk) = + mam4::utils::min_max_bound(0.001, 10.0, wsub(icol, kk)); + wsubice(icol, kk) = + mam4::utils::min_max_bound(0.2, 10.0, wsub(icol, kk)); + wsub(icol, kk) = haero::max(wsubmin, wsub(icol, kk)); + }); } void compute_subgrid_scale_velocities( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d wsub, - MAMAci::view_2d wsubice, - MAMAci::view_2d wsig, - MAMAci::const_view_2d tke, - const Real wsubmin, - const int top_lev, - const int nlev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_subgrid_scale_velocities(team, wsub, wsubice, wsig, tke, wsubmin, icol, top_lev, nlev); - }); + haero::ThreadTeamPolicy team_policy, MAMAci::view_2d wsub, + MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, + const Real wsubmin, const int top_lev, const int nlev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_subgrid_scale_velocities(team, wsub, wsubice, wsig, tke, + wsubmin, icol, top_lev, nlev); + }); } - KOKKOS_INLINE_FUNCTION -Real subgrid_mean_updraft(const Real w0, const Real wsig) -{ -/* --------------------------------------------------------------------------------- - Purpose: Calculate the mean updraft velocity inside a GCM grid assuming the - vertical velocity distribution is Gaussian and peaks at the - GCM resolved large-scale vertical velocity. - When icenul_wsub_scheme = 2, the model uses the mean updraft velocity as the - characteristic updraft velocity to calculate the ice nucleation rate. - Author: Kai Zhang (kai.zhang@pnnl.gov) - Last Modified: Oct, 2015 --------------------------------------------------------------------------------- */ - -// interface - -// in :: wsig standard deviation (m/s) -// in :: w0 large scale vertical velocity (m/s) -// out:: mean updraft velocity(m/s) -> characteristic w* +Real subgrid_mean_updraft(const Real w0, const Real wsig) { + /* --------------------------------------------------------------------------------- + Purpose: Calculate the mean updraft velocity inside a GCM grid assuming the + vertical velocity distribution is Gaussian and peaks at the + GCM resolved large-scale vertical velocity. + When icenul_wsub_scheme = 2, the model uses the mean updraft + velocity as the characteristic updraft velocity to calculate the ice + nucleation rate. Author: Kai Zhang (kai.zhang@pnnl.gov) Last Modified: Oct, + 2015 + -------------------------------------------------------------------------------- +*/ + + // interface + + // in :: wsig standard deviation (m/s) + // in :: w0 large scale vertical velocity (m/s) + // out:: mean updraft velocity(m/s) -> characteristic w* // FIXME should nbin be a user parameter? const int nbin = 50; - - using C = physics::Constants; - constexpr Real pi = C::Pi; + using C = physics::Constants; + constexpr Real pi = C::Pi; Real zz[nbin], wa = 0, ww = 0; int kp = 0; @@ -188,16 +166,19 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) const Real xx = 6.0 * sigma / nbin; - for (int ibin = 0; ibin < nbin; ++ibin) { - Real yy = wlarge - 3.0*sigma + 0.5*xx; - yy += (ibin-1)*xx; + for(int ibin = 0; ibin < nbin; ++ibin) { + Real yy = wlarge - 3.0 * sigma + 0.5 * xx; + yy += (ibin - 1) * xx; // wbar = integrator < w * f(w) * dw > - zz[ibin] = yy * haero::exp(-1.0*haero::square(yy-wlarge)/(2*sigma*sigma))/(sigma*haero::sqrt(2*pi))*xx; + zz[ibin] = + yy * + haero::exp(-1.0 * haero::square(yy - wlarge) / (2 * sigma * sigma)) / + (sigma * haero::sqrt(2 * pi)) * xx; } - for (int ibin = 0; ibin < nbin; ++ibin) { - if (zz[ibin] > 0) ++kp, wa += zz[ibin]; + for(int ibin = 0; ibin < nbin; ++ibin) { + if(zz[ibin] > 0) ++kp, wa += zz[ibin]; } - if (kp) { + if(kp) { // wbar = integrator < w * f(w) * dw > ww = wa; } else { @@ -206,254 +187,226 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) return ww; } KOKKOS_INLINE_FUNCTION -void compute_subgrid_mean_updraft_velocities( - const haero::ThreadTeam &team, - MAMAci::view_2d w2, - MAMAci::const_view_2d w0, - MAMAci::const_view_2d wsig, - const int icol, - const int nlev) -{ - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); - }); +void compute_subgrid_mean_updraft_velocities(const haero::ThreadTeam &team, + MAMAci::view_2d w2, + MAMAci::const_view_2d w0, + MAMAci::const_view_2d wsig, + const int icol, const int nlev) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + w2(icol, kk) = subgrid_mean_updraft(w0(icol, kk), wsig(icol, kk)); + }); } void compute_subgrid_mean_updraft_velocities( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d w2, - MAMAci::const_view_2d w0, - MAMAci::const_view_2d wsig, - const int nlev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_subgrid_mean_updraft_velocities(team, w2, w0, wsig, icol, nlev); - }); + haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w2, + MAMAci::const_view_2d w0, MAMAci::const_view_2d wsig, const int nlev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_subgrid_mean_updraft_velocities(team, w2, w0, wsig, icol, nlev); + }); } KOKKOS_INLINE_FUNCTION -void compute_aitken_dry_diameter( - const haero::ThreadTeam &team, - MAMAci::view_2d aitken_dry_dia, - MAMAci::const_view_3d dgnum, - const int icol, - const int top_lev) -{ +void compute_aitken_dry_diameter(const haero::ThreadTeam &team, + MAMAci::view_2d aitken_dry_dia, + MAMAci::const_view_3d dgnum, const int icol, + const int top_lev) { const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); - }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + aitken_dry_dia(icol, kk) = dgnum(icol, kk, aitken_idx); + }); } -void compute_aitken_dry_diameter( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d aitken_dry_dia, - MAMAci::const_view_3d dgnum, - const int top_lev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_aitken_dry_diameter(team, aitken_dry_dia, dgnum, icol, top_lev); - }); +void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d aitken_dry_dia, + MAMAci::const_view_3d dgnum, + const int top_lev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_aitken_dry_diameter(team, aitken_dry_dia, dgnum, icol, top_lev); + }); } void compute_nucleate_ice_tendencies( - const mam4::NucleateIce &nucleate_ice, - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d nihf, - MAMAci::view_2d niim, - MAMAci::view_2d nidep, - MAMAci::view_2d nimey, - MAMAci::view_2d naai_hom, - MAMAci::view_2d naai, + const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, + MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, mam_coupling::AerosolState &aerosol_state, - mam_coupling::DryAtmosphere &dry_atmosphere, - MAMAci::view_2d aitken_dry_dia, - const int nlev) -{ + mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d aitken_dry_dia, + const int nlev) { //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation //------------------------------------------------------------- - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - MAMAci::const_view_2d qv_dry = dry_atmosphere.qv; - MAMAci::const_view_2d cldfrac = dry_atmosphere.cldfrac; + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + MAMAci::const_view_2d qv_dry = dry_atmosphere.qv; + MAMAci::const_view_2d cldfrac = dry_atmosphere.cldfrac; MAMAci::const_view_2d w_updraft = dry_atmosphere.w_updraft; using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, - ekat::subview(T_mid, icol), - ekat::subview(p_mid, icol), - ekat::subview(qv_dry, icol), - dummy, dummy, dummy, dummy, dummy, dummy, dummy, - ekat::subview(cldfrac, icol), - ekat::subview(w_updraft, icol), pblh); - // set surface state data - haero::Surface surf{}; - mam4::Prognostics progs = mam_coupling::aerosols_for_column(aerosol_state, icol); - - // nucleation doesn't use any diagnostics, so it's okay to leave this alone - // for now - mam4::Diagnostics diags(nlev); - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - diags.dry_geometric_mean_diameter_i[aitken_idx] = ekat::subview(aitken_dry_dia, icol); - - // These are the fields that are updated. Taking subviews means that - // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated - // in nucleate_ice.compute_tendencies. - diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); - diags.icenuc_num_immfrz = ekat::subview(niim, icol); - diags.icenuc_num_depnuc = ekat::subview(nidep, icol); - diags.icenuc_num_meydep = ekat::subview(nimey, icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - // grab views from the buffer to store tendencies, not used as all values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t=0, dt=0; - /* - NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". - Only output we care about is "naai", "naai_hom" is never used anywhere - - Fortran code: - call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & ! input - rho, wsubice, strat_cld_frac, dgnum, & ! input - naai, naai_hom) ! output - */ - nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); - }); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + // Set up an atmosphere, surface, diagnostics, pronostics and tendencies + // class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), + ekat::subview(qv_dry, icol), dummy, dummy, dummy, dummy, dummy, + dummy, dummy, ekat::subview(cldfrac, icol), + ekat::subview(w_updraft, icol), pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(aerosol_state, icol); + + // nucleation doesn't use any diagnostics, so it's okay to leave this + // alone for now + mam4::Diagnostics diags(nlev); + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + diags.dry_geometric_mean_diameter_i[aitken_idx] = + ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated + // in nucleate_ice.compute_tendencies. + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0, dt = 0; + /* + NOTE:"state_q" is a combination of subset of tracers added by + "int_mmr_field_name" and "int_nmr_field_name". Only output we care + about is "naai", "naai_hom" is never used anywhere + + Fortran code: + call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & + ! input rho, wsubice, strat_cld_frac, dgnum, & ! input naai, + naai_hom) ! output + */ + nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, + progs, diags, tends); + }); } KOKKOS_INLINE_FUNCTION -void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - MAMAci::const_view_2d qc, - MAMAci::const_view_2d qi, - MAMAci::const_view_2d liqcldf, - const int icol, - const int top_lev) -{ +void store_liquid_cloud_fraction(const haero::ThreadTeam &team, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + MAMAci::const_view_2d qc, + MAMAci::const_view_2d qi, + MAMAci::const_view_2d liqcldf, const int icol, + const int top_lev) { //------------------------------------------------------------- - // Get old and new liquid cloud fractions when amount of cloud + // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value - + // MUST FIXME NOTE: We need old and new liquid cloud fractions here. - // We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to - // store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make - // a note of it and use the new cloud fraction for the - // old cloud fraction. + // We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to + // store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we + // will make a note of it and use the new cloud fraction for the old cloud + // fraction. //------------------------------------------------------------- - static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - const Real qcld = qc(icol,kk) + qi(icol,kk); - if (qcld > qsmall) { - cloud_frac_new(icol,kk)=liqcldf(icol,kk); - cloud_frac_old(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old - } else { - cloud_frac_new(icol,kk)=0; - cloud_frac_old(icol,kk)=0; - } - }); + static constexpr auto qsmall = + 1e-18; // cut-off for cloud amount (ice or liquid) + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + const Real qcld = qc(icol, kk) + qi(icol, kk); + if(qcld > qsmall) { + cloud_frac_new(icol, kk) = liqcldf(icol, kk); + cloud_frac_old(icol, kk) = + liqcldf(icol, kk); // FIXME should be liqcldf_old + } else { + cloud_frac_new(icol, kk) = 0; + cloud_frac_old(icol, kk) = 0; + } + }); } -void store_liquid_cloud_fraction( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - mam_coupling::WetAtmosphere &wet_atmosphere, - MAMAci::const_view_2d liqcldf, - const int top_lev) -{ +void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::const_view_2d liqcldf, + const int top_lev) { MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d qi = wet_atmosphere.qi; - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, qi, liqcldf, icol, top_lev); - }); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, + qi, liqcldf, icol, top_lev); + }); } KOKKOS_INLINE_FUNCTION -void compute_recipical_pseudo_density( - const haero::ThreadTeam &team, - MAMAci::view_2d rpdel, - MAMAci::const_view_2d pdel, - const int icol, - const int nlev) -{ - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - rpdel(icol,kk) = 1/pdel(icol,kk); - }); +void compute_recipical_pseudo_density(const haero::ThreadTeam &team, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d pdel, + const int icol, const int nlev) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev), + KOKKOS_LAMBDA(int kk) { rpdel(icol, kk) = 1 / pdel(icol, kk); }); } -void compute_recipical_pseudo_density( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d rpdel, - MAMAci::const_view_2d pdel, - const int nlev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_recipical_pseudo_density(team, rpdel, pdel, icol, nlev); - }); +void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d pdel, + const int nlev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_recipical_pseudo_density(team, rpdel, pdel, icol, nlev); + }); } void call_function_dropmixnuc( - haero::ThreadTeamPolicy team_policy, - mam_coupling::DryAtmosphere &dry_atmosphere, - const Real dtmicro, - MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], - MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], - MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], - MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], - MAMAci::const_view_2d p_int, - MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, - MAMAci::const_view_3d state_q, - MAMAci::const_view_2d ncldwtr, - MAMAci::const_view_2d kvh, - MAMAci::view_2d qcld, - MAMAci::view_2d wsub, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - MAMAci::view_2d tendnd, - MAMAci::view_3d factnum, - MAMAci::view_2d ndropcol, - MAMAci::view_2d ndropmix, - MAMAci::view_2d nsource, - MAMAci::view_2d wtke, - MAMAci::view_3d ccn, - MAMAci::view_3d nact, - MAMAci::view_3d mact, - MAMAci::view_2d dropmixnuc_scratch_mem[15], - const int nlev) -{ + haero::ThreadTeamPolicy team_policy, + mam_coupling::DryAtmosphere &dry_atmosphere, const Real dtmicro, + MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], + MAMAci::view_2d raercol[mam4::ndrop::pver][2], + MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], + MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], + MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, + MAMAci::view_2d rpdel, MAMAci::const_view_3d state_q, + MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, + MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, + MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, + MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, + MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - MAMAci::const_view_2d zm = dry_atmosphere.z_mid; - - MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; - MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; - MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; - MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; - MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; - MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; + MAMAci::const_view_2d zm = dry_atmosphere.z_mid; + + MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; + MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; + MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; + MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; + MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; + MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; - MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; - MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; - MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; - MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; - MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; - MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; - MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; + MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; + MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; + MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; + MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; + MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; + MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; + MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; @@ -462,622 +415,703 @@ void call_function_dropmixnuc( MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; - for (int i=0; i -void copy_mam4xx_array_to_scream( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_3d scream, - MAMAci::view_2d mam4xx[len], - const int nlev) -{ +template +void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, + MAMAci::view_3d scream, + MAMAci::view_2d mam4xx[len], const int nlev) { // Localize the input arrays. MAMAci::view_2d mam4xx_loc[len]; - for (int view_num=0; view_num < len; ++view_num) + for(int view_num = 0; view_num < len; ++view_num) mam4xx_loc[view_num] = mam4xx[view_num]; - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - for (int view_num=0; view_num < len; ++view_num) { - MAMAci::view_2d mam4xx_view = mam4xx_loc[view_num]; - copy_mam4xx_array_to_scream(team, scream, mam4xx_view, icol, nlev, view_num); - } - }); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + for(int view_num = 0; view_num < len; ++view_num) { + MAMAci::view_2d mam4xx_view = mam4xx_loc[view_num]; + copy_mam4xx_array_to_scream(team, scream, mam4xx_view, icol, nlev, + view_num); + } + }); } void call_hetfrz_compute_tendencies( - haero::ThreadTeamPolicy team_policy, - mam4::Hetfrz &hetfrz_, - mam_coupling::AerosolState &dry_aero_, - mam_coupling::WetAtmosphere &wet_atmosphere_, - mam_coupling::DryAtmosphere &dry_atmosphere_, - MAMAci::view_2d stratiform_cloud_fraction, - MAMAci::view_2d activation_fraction_accum_idx, - MAMAci::view_2d activation_fraction_coarse_idx, - MAMAci::view_2d hetfrz_immersion_nucleation_tend, - MAMAci::view_2d hetfrz_contact_nucleation_tend, - MAMAci::view_2d hetfrz_depostion_nucleation_tend, - MAMAci::view_2d naai_hom, - MAMAci::view_2d naai, - MAMAci::view_2d diagnostic_scratch_[], - const int nlev) -{ + haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, + mam_coupling::AerosolState &dry_aero_, + mam_coupling::WetAtmosphere &wet_atmosphere_, + mam_coupling::DryAtmosphere &dry_atmosphere_, + MAMAci::view_2d stratiform_cloud_fraction, + MAMAci::view_2d activation_fraction_accum_idx, + MAMAci::view_2d activation_fraction_coarse_idx, + MAMAci::view_2d hetfrz_immersion_nucleation_tend, + MAMAci::view_2d hetfrz_contact_nucleation_tend, + MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, + MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], + const int nlev) { using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); - mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState aerosol_state = dry_aero_; - mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState aerosol_state = dry_aero_; + mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; MAMAci::view_2d diagnostic_scratch[42]; - for (int i=0; i<42; ++i) - diagnostic_scratch[i] = diagnostic_scratch_[i];; + for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; + ; - MAMAci::const_view_2d qc = wet_atmosphere.qc; - MAMAci::const_view_2d nc = wet_atmosphere.nc; + MAMAci::const_view_2d qc = wet_atmosphere.qc; + MAMAci::const_view_2d nc = wet_atmosphere.nc; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, - ekat::subview(T_mid, icol), - ekat::subview(p_mid, icol), - dummy, - ekat::subview(qc, icol), - ekat::subview(nc, icol), - dummy, dummy, dummy, dummy, dummy, dummy, dummy, - pblh); - // set surface state data - haero::Surface surf{}; - mam4::Prognostics progs = mam_coupling::aerosols_for_column(aerosol_state, icol); - - const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - - mam4::Diagnostics diags(nlev); - diags.stratiform_cloud_fraction = ekat::subview(stratiform_cloud_fraction, icol); - diags.activation_fraction[accum_idx] = ekat::subview(activation_fraction_accum_idx, icol); - diags.activation_fraction[coarse_idx] = ekat::subview(activation_fraction_coarse_idx, icol); - - // These are the output tendencies from heterogeneous freezing that need to be - // added correctly to the cloud-micorphysics scheme. - diags.hetfrz_immersion_nucleation_tend = ekat::subview(hetfrz_immersion_nucleation_tend, icol); - diags.hetfrz_contact_nucleation_tend = ekat::subview(hetfrz_contact_nucleation_tend, icol); - diags.hetfrz_depostion_nucleation_tend = ekat::subview(hetfrz_depostion_nucleation_tend, icol); - - diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); - diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); - diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); - diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); - diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); - diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); - diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); - diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); - diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); - diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); - diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); - diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); - diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); - diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); - diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); - diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); - diags.na500 = ekat::subview(diagnostic_scratch[18], icol); - diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); - diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); - diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); - diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); - diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); - diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); - diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); - diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); - diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); - diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); - diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); - diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); - diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); - diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); - diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); - diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); - diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); - diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); - diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); - diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); - diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); - diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - //------------------------------------------------------------- - // Heterogeneous freezing - // frzimm, frzcnt, frzdep are the outputs of - // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) - //------------------------------------------------------------- - // - // grab views from the buffer to store tendencies, not used as all values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t=0, dt=0; - hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); - }); -} + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + // Set up an atmosphere, surface, diagnostics, pronostics and tendencies + // class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, + ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, + dummy, dummy, dummy, dummy, dummy, pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(aerosol_state, icol); + + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + + mam4::Diagnostics diags(nlev); + diags.stratiform_cloud_fraction = + ekat::subview(stratiform_cloud_fraction, icol); + diags.activation_fraction[accum_idx] = + ekat::subview(activation_fraction_accum_idx, icol); + diags.activation_fraction[coarse_idx] = + ekat::subview(activation_fraction_coarse_idx, icol); + + // These are the output tendencies from heterogeneous freezing that need + // to be added correctly to the cloud-micorphysics scheme. + diags.hetfrz_immersion_nucleation_tend = + ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = + ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = + ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0, dt = 0; + hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, + diags, tends); + }); } +} // namespace -//FIXME: The following variables are namelist variables +// FIXME: The following variables are namelist variables const Real wsubmin = 1; -MAMAci::MAMAci( - const ekat::Comm& comm, - const ekat::ParameterList& params) - : AtmosphereProcess(comm, params){ -} +MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) {} - -//Return type of the process -AtmosphereProcessType MAMAci::type() const{ +// Return type of the process +AtmosphereProcessType MAMAci::type() const { return AtmosphereProcessType::Physics; } -//return name of the process -std::string MAMAci::name() const{ - return "mam4_aci"; - } +// return name of the process +std::string MAMAci::name() const { return "mam4_aci"; } -//set grid for all the inputs and outputs -void MAMAci::set_grids(const std::shared_ptr grids_manager) { - m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI set grid"); +// set grid for all the inputs and outputs +void MAMAci::set_grids( + const std::shared_ptr grids_manager) { + m_atm_logger->log(ekat::logger::LogLevel::info, "Calling ACI set grid"); - grid_ = grids_manager->get_grid("Physics"); //Use physics grid - const auto& grid_name = grid_->name(); + grid_ = grids_manager->get_grid("Physics"); // Use physics grid + const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - Kokkos::resize(rho_, ncol_, nlev_); - Kokkos::resize(w0_, ncol_, nlev_); - Kokkos::resize(tke_, ncol_, nlev_+1); - Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + Kokkos::resize(tke_, ncol_, nlev_ + 1); + Kokkos::resize(wsub_, ncol_, nlev_); Kokkos::resize(wsubice_, ncol_, nlev_); - Kokkos::resize(wsig_, ncol_, nlev_); - Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(cloud_frac_new_, ncol_, nlev_); - Kokkos::resize(cloud_frac_old_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(cloud_frac_new_, ncol_, nlev_); + Kokkos::resize(cloud_frac_old_, ncol_, nlev_); Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); - Kokkos::resize(rpdel_, ncol_, nlev_); + Kokkos::resize(rpdel_, ncol_, nlev_); - for (int i=0; i<15; ++i) { - Kokkos::resize(dropmixnuc_scratch_mem_[i],ncol_, nlev_); + for(int i = 0; i < 15; ++i) { + Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; // Layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points - FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces - - ekat::units::Units kg = ekat::units::kg; - ekat::units::Units Pa = ekat::units::Pa; - ekat::units::Units s = ekat::units::s; - ekat::units::Units m = ekat::units::m; - ekat::units::Units K = ekat::units::K; - auto q_unit = kg/kg; // units of mass mixing ratios of tracers + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // mid points + FieldLayout scalar3d_layout_int{{COL, ILEV}, + {ncol_, nlev_ + 1}}; // interfaces + + ekat::units::Units kg = ekat::units::kg; + ekat::units::Units Pa = ekat::units::Pa; + ekat::units::Units s = ekat::units::s; + ekat::units::Units m = ekat::units::m; + ekat::units::Units K = ekat::units::K; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); - auto n_unit = 1/kg; // units of number mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); auto nondim = ekat::units::Units::nondimensional(); // atmospheric quantities - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // specific humidity [kg/kg] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud ice number mixing ratio [1/kg] - add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints - add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints - add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces - add_field ("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // - // - add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - - add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] - add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] - //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity [kg/kg] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // cloud liquid mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // cloud ice mass mixing ratio [kg/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud ice number mixing ratio [1/kg] + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // Temperature[K] at midpoints + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // Total pressure [Pa] at midpoints + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // Total pressure [Pa] at interfaces + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // Layer thickness(pdel) [Pa] at midpoints + // + // + add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, + grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, + grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, + grid_name); // Layer thickness(pdel) [Pa] at midpoints + + add_field( + "state_q", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars}}, q_unit, + grid_name); // aerosol mmrs [kg/kg] + add_field("ncldwtr", scalar3d_layout_mid, n_unit, + grid_name); // initial droplet number mixing ratio [#/kg] + // add_field("cldo", , unitless, grid_name); // cloud fraction on + // previous time step [fraction] // - add_field ("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] - add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] - - - add_field("icenuc_num_hetfrz",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] - add_field("icenuc_num_immfrz",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] - add_field("icenuc_num_depnuc",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] - add_field("icenuc_num_meydep",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] - add_field("num_act_aerosol_ice_nucle_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] - add_field("num_act_aerosol_ice_nucle",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] - add_field("qcld",scalar3d_layout_mid , n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] - add_field("ptend_q", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne aerosols [#/kg] - add_field("tendnd",scalar3d_layout_mid , n_unit/s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] - add_field("factnum", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam_coupling::num_aero_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] - add_field("ndropcol",scalar3d_layout_mid , n_unit/s, grid_name); // - add_field("ndropmix",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio tendency due to mixing [#/kg/s] - add_field("nsource",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] - add_field("wtke",scalar3d_layout_mid , n_unit/s, grid_name); // - add_field("ccn", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, grid_name); //number conc of aerosols activated at supersat [#/m^3] - // note: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud [#/m^2/s]] - // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] - add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output - add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency - - - //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { - //interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - - //cloudborne aerosol tracers of interest: number (n) mixing ratios - //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + add_field("w_updraft", scalar3d_layout_mid, q_unit, + grid_name); // updraft velocity [m/s] + add_field( + "qqcw", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, + q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios + // [#/kg or kg/kg] + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + grid_name); // cloud fraction [nondimentional] + + add_field("icenuc_num_hetfrz", scalar3d_layout_mid, 1 / m / m / m, + grid_name); // number conc of ice nuclei due to + // heterogeneous freezing [1/m3] + add_field("icenuc_num_immfrz", scalar3d_layout_mid, 1 / m / m / m, + grid_name); // number conc of ice nuclei due to immersion + // freezing (hetero nuc) [1/m3] + add_field("icenuc_num_depnuc", scalar3d_layout_mid, 1 / m / m / m, + grid_name); // number conc of ice nuclei due to + // deposition nucleation (hetero nuc)[1/m3] + add_field( + "icenuc_num_meydep", scalar3d_layout_mid, 1 / m / m / m, + grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + add_field( + "num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, n_unit, + grid_name); // number of activated aerosol for ice nucleation + // (homogeneous freezing only) [#/kg] + add_field( + "num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, + grid_name); // number of activated aerosol for ice nucleation[#/kg] + add_field("qcld", scalar3d_layout_mid, n_unit, + grid_name); // cloud droplet number mixing ratio [#/kg] + add_field( + "ptend_q", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, + n_unit, grid_name); // tendencies for interstitial and cloud borne + // aerosols [#/kg] + add_field( + "tendnd", scalar3d_layout_mid, n_unit / s, + grid_name); // tendency in droplet number mixing ratio [#/kg/s] + add_field( + "factnum", + FieldLayout{{COL, LEV, CMP}, + {ncol_, nlev_, mam_coupling::num_aero_modes()}}, + nondim, grid_name); // activation fraction for aerosol number [fraction] + add_field("ndropcol", scalar3d_layout_mid, n_unit / s, + grid_name); // + add_field("ndropmix", scalar3d_layout_mid, n_unit / s, + grid_name); // droplet number mixing ratio tendency due + // to mixing [#/kg/s] + add_field( + "nsource", scalar3d_layout_mid, n_unit / s, + grid_name); // droplet number mixing ratio source tendency [#/kg/s] + add_field("wtke", scalar3d_layout_mid, n_unit / s, grid_name); // + add_field( + "ccn", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, + n_unit, + grid_name); // number conc of aerosols activated at supersat [#/m^3] + // note: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud + // [#/m^2/s]] + // / [aero. number conc. in updraft, just below + // cloudbase [#/m^3]] + add_field( + "coltend", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, + nondim, grid_name); // column tendency for diagnostic output + add_field( + "coltend_cw", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, + nondim, grid_name); // column tendency + + // MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to + // dry + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); - if (strlen(int_mmr_field_name) > 0) - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(mode, a); + if(strlen(int_mmr_field_name) > 0) + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); - if (strlen(cld_mmr_field_name) > 0) - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(mode, a); + if(strlen(cld_mmr_field_name) > 0) + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); } } - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); } - //Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM - //These outputs should come from the cloud macrophysics process (e.g., SHOC) - auto m2 = m*m; + // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM + // These outputs should come from the cloud macrophysics process (e.g., SHOC) + auto m2 = m * m; m2.set_string("m^2"); - auto s2 = s*s; + auto s2 = s * s; s2.set_string("s^2"); - //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time - add_field("w_sec", scalar3d_layout_int, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints + // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and + // liq_strat_cld_frac may also need OLD time + add_field( + "w_sec", scalar3d_layout_int, m2 / s2, + grid_name); // Vertical velocity variance (wp2) at midpoints + + add_field("strat_cld_frac", scalar3d_layout_mid, nondim, + grid_name); // Stratiform cloud fraction at midpoints + add_field( + "liq_strat_cld_frac", scalar3d_layout_mid, nondim, + grid_name); // Liquid stratiform cloud fraction at midpoints + add_field("kvh", scalar3d_layout_int, m2 / s, + grid_name); // Eddy diffusivity for heat - add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints - add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints - add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes} }; // mid points - add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols - - - auto cm = m/100; - auto frz_unit = 1/(cm*cm*cm*s); // units of number mixing ratios of tracers + FieldLayout scalar4d_layout_mid{ + {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + add_field("dgnum", scalar4d_layout_mid, m, + grid_name); // dry diameter of aerosols + + auto cm = m / 100; + auto frz_unit = + 1 / (cm * cm * cm * s); // units of number mixing ratios of tracers n_unit.set_string("1(cm^-3 s^-1)"); - add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols - add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols - add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, + frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, + frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, + frz_unit, grid_name); // dry diameter of aerosols // - /* + /* * NOTE on other inputs for the aci process: * 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density - * 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using - * pseudo_density, p_mid, T_mid and qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). + * 2. geopotential height at midpoints: computed geopotential height at + * interfaces, which inturn is computed using pseudo_density, p_mid, T_mid and + * qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). * qv_mid can be obtained from "get_field_in" call */ - } -size_t MAMAci::requested_buffer_size_in_bytes() const -{ +size_t MAMAci::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); - - size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); - EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), - "Error! Used memory != requested memory for MAMMicrophysics."); + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMMicrophysics."); } - void MAMAci::initialize_impl(const RunType run_type) { - m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); + m_atm_logger->log(ekat::logger::LogLevel::info, "Calling ACI init"); // set atmosphere state data - - p_int_ = get_field_in("p_int").get_view(); - w_sec_ = get_field_in("w_sec").get_view(); - state_q_ = get_field_in("state_q").get_view(); - ncldwtr_ = get_field_in("ncldwtr").get_view(); - qqcw_input_ = get_field_in("qqcw").get_view(); - dgnum_ = get_field_in("dgnum").get_view(); - nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); - niim_ = get_field_out("icenuc_num_immfrz").get_view(); - nidep_ = get_field_out("icenuc_num_depnuc").get_view(); - nimey_ = get_field_out("icenuc_num_meydep").get_view(); - naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); - naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); - qcld_ = get_field_out("qcld").get_view(); - ptend_q_output_ = get_field_out("ptend_q").get_view(); - tendnd_ = get_field_out("tendnd").get_view(); - factnum_ = get_field_out("factnum").get_view(); - ndropcol_ = get_field_out("ndropcol").get_view(); - ndropmix_ = get_field_out("ndropmix").get_view(); - nsource_ = get_field_out("nsource").get_view(); - wtke_ = get_field_out("wtke").get_view(); - ccn_ = get_field_out("ccn").get_view(); - coltend_outp_ = get_field_out("coltend").get_view(); - coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); - kvh_ = get_field_in("kvh").get_view(); - - wet_atmosphere_.qv = get_field_in("qv").get_view(); - wet_atmosphere_.qc = get_field_in("qc").get_view(); - wet_atmosphere_.nc = get_field_in("nc").get_view(); - wet_atmosphere_.qi = get_field_in("qi").get_view(); - wet_atmosphere_.ni = get_field_in("ni").get_view(); - wet_atmosphere_.omega = get_field_in("omega").get_view(); - - dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); - dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); - dry_atmosphere_.p_del = get_field_in("pseudo_density").get_view(); - dry_atmosphere_.qv = buffer_.qv_dry; - dry_atmosphere_.qc = buffer_.qc_dry; - dry_atmosphere_.nc = buffer_.nc_dry; - dry_atmosphere_.qi = buffer_.qi_dry; - dry_atmosphere_.ni = buffer_.ni_dry; - - dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) - dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above surface at interface levels (m) - dry_atmosphere_.z_mid = buffer_.z_mid; // geopotential height above surface at mid levels (m) - - dry_atmosphere_.cldfrac = get_field_in("cldfrac_tot").get_view(); - dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); - - hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); - hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); - - stratiform_cloud_fraction_ = get_field_out("stratiform_cloud_fraction").get_view(); - activation_fraction_accum_idx_ = get_field_out("activation_fraction_accum").get_view(); - activation_fraction_coarse_idx_ = get_field_out("activation_fraction_coarse").get_view(); - - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - auto prog_index = [](const int mode, const int species) { + + p_int_ = get_field_in("p_int").get_view(); + w_sec_ = get_field_in("w_sec").get_view(); + state_q_ = get_field_in("state_q").get_view(); + ncldwtr_ = get_field_in("ncldwtr").get_view(); + qqcw_input_ = get_field_in("qqcw").get_view(); + dgnum_ = get_field_in("dgnum").get_view(); + nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); + niim_ = get_field_out("icenuc_num_immfrz").get_view(); + nidep_ = get_field_out("icenuc_num_depnuc").get_view(); + nimey_ = get_field_out("icenuc_num_meydep").get_view(); + naai_hom_ = + get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); + naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); + qcld_ = get_field_out("qcld").get_view(); + ptend_q_output_ = get_field_out("ptend_q").get_view(); + tendnd_ = get_field_out("tendnd").get_view(); + factnum_ = get_field_out("factnum").get_view(); + ndropcol_ = get_field_out("ndropcol").get_view(); + ndropmix_ = get_field_out("ndropmix").get_view(); + nsource_ = get_field_out("nsource").get_view(); + wtke_ = get_field_out("wtke").get_view(); + ccn_ = get_field_out("ccn").get_view(); + coltend_outp_ = get_field_out("coltend").get_view(); + coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + kvh_ = get_field_in("kvh").get_view(); + + wet_atmosphere_.qv = get_field_in("qv").get_view(); + wet_atmosphere_.qc = get_field_in("qc").get_view(); + wet_atmosphere_.nc = get_field_in("nc").get_view(); + wet_atmosphere_.qi = get_field_in("qi").get_view(); + wet_atmosphere_.ni = get_field_in("ni").get_view(); + wet_atmosphere_.omega = get_field_in("omega").get_view(); + + dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); + dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); + dry_atmosphere_.p_del = + get_field_in("pseudo_density").get_view(); + dry_atmosphere_.qv = buffer_.qv_dry; + dry_atmosphere_.qc = buffer_.qc_dry; + dry_atmosphere_.nc = buffer_.nc_dry; + dry_atmosphere_.qi = buffer_.qi_dry; + dry_atmosphere_.ni = buffer_.ni_dry; + + dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) + dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above + // surface at interface levels (m) + dry_atmosphere_.z_mid = + buffer_.z_mid; // geopotential height above surface at mid levels (m) + + dry_atmosphere_.cldfrac = + get_field_in("cldfrac_tot").get_view(); + dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); + + hetfrz_immersion_nucleation_tend_ = + get_field_out("hetfrz_immersion_nucleation_tend").get_view(); + hetfrz_contact_nucleation_tend_ = + get_field_out("hetfrz_contact_nucleation_tend").get_view(); + hetfrz_depostion_nucleation_tend_ = + get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + + stratiform_cloud_fraction_ = + get_field_out("stratiform_cloud_fraction").get_view(); + activation_fraction_accum_idx_ = + get_field_out("activation_fraction_accum").get_view(); + activation_fraction_coarse_idx_ = + get_field_out("activation_fraction_coarse").get_view(); + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + auto prog_index = [](const int mode, const int species) { const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); - const int ind = - (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; + const int ind = + (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; return ind; }; - //interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; - //cloudborne aerosol tracers of interest: number (n) mixing ratios - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { const int index = prog_index(m, a); - wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - std::cout<<"BALLI:-----"<(); + std::cout << "BALLI:-----" << m << " " << a << " " << index << " " + << wet_aero_.int_aero_mmr[m][a](0, 0) << std::endl; dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } - + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { const int index = prog_index(m, a); - wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } - for (int i=0; ilog(ekat::logger::LogLevel::info,"calling ACI run"); + m_atm_logger->log(ekat::logger::LogLevel::info, "calling ACI run"); const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -1109,45 +1144,47 @@ void MAMAci::run_impl(const double dt) { // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion - compute_w0_and_rho(team_policy, w0_, rho_, wet_atmosphere_, dry_atmosphere_, top_lev_, nlev_); + compute_w0_and_rho(team_policy, w0_, rho_, wet_atmosphere_, dry_atmosphere_, + top_lev_, nlev_); compute_tke_using_w_sec(team_policy, tke_, w_sec_, nlev_); - Kokkos::fence(); // wait for for tke_ to be computed. + Kokkos::fence(); // wait for for tke_ to be computed. + + compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, + wsubmin, top_lev_, nlev_); + Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, wsubmin, top_lev_, nlev_); - Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); - + compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); - Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, - nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, - dry_aero_, dry_atmosphere_, aitken_dry_dia_, nlev_); + compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, + nidep_, nimey_, naai_hom_, naai_, dry_aero_, + dry_atmosphere_, aitken_dry_dia_, nlev_); - store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, wet_atmosphere_, liqcldf_, top_lev_); + store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, + wet_atmosphere_, liqcldf_, top_lev_); //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous + // Save cloud borne aerosols to be used in the heterozenous // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne + // process. This is only a select subset of cloud borne // aerosols, not all the cloud borne aerosols. //------------------------------------------------------------- - /*NOTE: We probably need to store indices for the select few cloud borne aerosols - Fortran code: - lchnk_zb = lchnk - begchunk - ! save copy of cloud borne aerosols for use in heterogeneous freezing before - ! we change it in dropmixnuc + /*NOTE: We probably need to store indices for the select few cloud borne + aerosols Fortran code: lchnk_zb = lchnk - begchunk ! save copy of cloud borne + aerosols for use in heterogeneous freezing before ! we change it in dropmixnuc do ispec = 1, ncnst call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) - aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * rho(:ncol,:) - enddo + aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * + rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, nlev_); - Kokkos::fence(); // wait for rpdel_ to be computed. + compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, + nlev_); + Kokkos::fence(); // wait for rpdel_ to be computed. #if 0 call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, @@ -1173,8 +1210,8 @@ void MAMAci::run_impl(const double dt) { #endif } -void MAMAci::finalize_impl(){ - m_atm_logger->log(ekat::logger::LogLevel::info,"calling ACI final"); +void MAMAci::finalize_impl() { + m_atm_logger->log(ekat::logger::LogLevel::info, "calling ACI final"); } -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 224314c5099..0523890cff8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -1,38 +1,37 @@ #ifndef EAMXX_MAM_ACI_HPP #define EAMXX_MAM_ACI_HPP -//For MAM4 aerosol configuration +// For MAM4 aerosol configuration #include -//For declaring ACI class derived from atm process class +// For declaring ACI class derived from atm process class #include -//For physical constants +// For physical constants #include "physics/share/physics_constants.hpp" -namespace scream -{ +namespace scream { class MAMAci final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; mam4::Hetfrz hetfrz_; // views for single- and multi-column data - using view_1d = scream::mam_coupling::view_1d; - using view_2d = scream::mam_coupling::view_2d; - using view_3d = scream::mam_coupling::view_3d; + using view_1d = scream::mam_coupling::view_1d; + using view_2d = scream::mam_coupling::view_2d; + using view_3d = scream::mam_coupling::view_3d; using const_view_1d = scream::mam_coupling::const_view_1d; using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; template - using view_4d = KT::view; + using view_4d = KT::view; -//FIXME:B: Should the following variables be public? They are like that in micriphysics and optics codes - // FIXME the time step for microphysics [s] need to get from the input + // FIXME:B: Should the following variables be public? They are like that in + // micriphysics and optics codes + // FIXME the time step for microphysics [s] need to get from the input const Real dtmicro_ = .0001; // rho is air density [kg/m3] @@ -46,7 +45,8 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d aitken_dry_dia_; - view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()][mam_coupling::num_aero_species()]; + view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()] + [mam_coupling::num_aero_species()]; mam_coupling::WetAtmosphere wet_atmosphere_; @@ -60,7 +60,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d naai_hom_; view_2d naai_; const_view_2d liqcldf_; - const_view_2d kvh_; + const_view_2d kvh_; const_view_3d state_q_; const_view_2d ncldwtr_; @@ -85,7 +85,8 @@ class MAMAci final : public scream::AtmosphereProcess { view_3d coltend_cw_outp_; view_2d coltend_cw_[mam4::ndrop::ncnst_tot]; - // raercol_cw_ and raercol_ are work arrays for dropmixnuc, allocated on the stack. + // raercol_cw_ and raercol_ are work arrays for dropmixnuc, allocated on the + // stack. view_2d raercol_cw_[mam4::ndrop::pver][2]; view_2d raercol_[mam4::ndrop::pver][2]; @@ -111,20 +112,33 @@ class MAMAci final : public scream::AtmosphereProcess { const int top_lev_ = 6; // local atmospheric state column variables - const_view_2d p_int_; // Total pressure [Pa] at interfaces - const_view_2d pdel_; // pressure thickess of layer [Pa] - view_2d rpdel_; // Inverse of pdel_ - const_view_2d w_sec_; // Vertical velocity variance + const_view_2d p_int_; // Total pressure [Pa] at interfaces + const_view_2d pdel_; // pressure thickess of layer [Pa] + view_2d rpdel_; // Inverse of pdel_ + const_view_2d w_sec_; // Vertical velocity variance + + // number of horizontal columns and vertical levels + int ncol_, nlev_; -public: + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // workspace manager for internal local variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + public: // Constructor - MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); + MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // process metadata AtmosphereProcessType type() const override; std::string name() const override; // grid - void set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -135,12 +149,6 @@ class MAMAci final : public scream::AtmosphereProcess { void run_impl(const double dt) override; void finalize_impl() override; - - //Local variables - - // number of horizontal columns and vertical levels - int ncol_, nlev_; - // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. @@ -148,11 +156,10 @@ class MAMAci final : public scream::AtmosphereProcess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere& wet_atm, - const mam_coupling::AerosolState& wet_aero, - const mam_coupling::DryAtmosphere& dry_atm, - const mam_coupling::AerosolState& dry_aero) { - + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { ncol_pre_ = ncol; nlev_pre_ = nlev; wet_atm_pre_ = wet_atm; @@ -167,9 +174,11 @@ class MAMAci final : public scream::AtmosphereProcess { const int i = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); team.team_barrier(); - //vertical heights has to be computed after computing dry mixing ratios for atmosphere + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere compute_vertical_layer_heights(team, dry_atm_pre_, i); } // operator() @@ -181,24 +190,15 @@ class MAMAci final : public scream::AtmosphereProcess { // local atmospheric and aerosol state data mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; - mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess - + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMAci::Preprocess + private: // pre- and postprocessing scratch pads Preprocess preprocess_; - // aerosol state variables - mam_coupling::AerosolState wet_aero_, dry_aero_; - - // workspace manager for internal local variables - mam_coupling::Buffer buffer_; - - // physics grid for column information - std::shared_ptr grid_; -}; // MAMAci - -} // namespace scream +}; // MAMAci +} // namespace scream -#endif // EAMXX_MAM_ACI_HPP +#endif // EAMXX_MAM_ACI_HPP From 93fb19227868c8cfe2e03ae92f20b5826114bed8 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 14 Mar 2024 21:37:21 -0700 Subject: [PATCH 029/142] Adds some temp comments and comment out state_q and qqcw inputs --- .../mam/eamxx_mam_aci_process_interface.cpp | 149 +++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 16 +- 2 files changed, 89 insertions(+), 76 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4b322245c3f..0b7fb739e7c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -232,7 +232,7 @@ void compute_nucleate_ice_tendencies( const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, - mam_coupling::AerosolState &aerosol_state, + mam_coupling::AerosolState &dry_aerosol_state, mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d aitken_dry_dia, const int nlev) { //------------------------------------------------------------- @@ -261,7 +261,7 @@ void compute_nucleate_ice_tendencies( // set surface state data haero::Surface surf{}; mam4::Prognostics progs = - mam_coupling::aerosols_for_column(aerosol_state, icol); + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); // nucleation doesn't use any diagnostics, so it's okay to leave this // alone for now @@ -355,6 +355,7 @@ void compute_recipical_pseudo_density(const haero::ThreadTeam &team, MAMAci::view_2d rpdel, MAMAci::const_view_2d pdel, const int icol, const int nlev) { + // FIXME: Add an assert to ensure pdel is non-zero Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { rpdel(icol, kk) = 1 / pdel(icol, kk); }); @@ -490,8 +491,9 @@ void call_function_dropmixnuc( ptend, nctend_mixnuc, factnum) !out */ - //mam4::utils::extract_stateq_from_prognostics(progs, atm, state_q_at_kk, - // kk); + // mam4::utils::extract_stateq_from_prognostics(progs, atm, + // state_q_at_kk, + // kk); mam4::ndrop::dropmixnuc( team, dtmicro, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), @@ -570,10 +572,10 @@ void call_hetfrz_compute_tendencies( using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); - mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState aerosol_state = dry_aero_; - mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState dry_aerosol_state = dry_aero_; + mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; ; @@ -597,7 +599,7 @@ void call_hetfrz_compute_tendencies( // set surface state data haero::Surface surf{}; mam4::Prognostics progs = - mam_coupling::aerosols_for_column(aerosol_state, icol); + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); @@ -690,14 +692,6 @@ const Real wsubmin = 1; MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) {} -// Return type of the process -AtmosphereProcessType MAMAci::type() const { - return AtmosphereProcessType::Physics; -} - -// return name of the process -std::string MAMAci::name() const { return "mam4_aci"; } - // set grid for all the inputs and outputs void MAMAci::set_grids( const std::shared_ptr grids_manager) { @@ -709,7 +703,9 @@ void MAMAci::set_grids( ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize( + rho_, ncol_, + nlev_); // BQ: Does Kokkos::resize allocates memory as well for rho_? Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -732,7 +728,7 @@ void MAMAci::set_grids( FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; // interfaces - ekat::units::Units kg = ekat::units::kg; + ekat::units::Units kg = ekat::units::kg; // BQ: why do we need to do this??? ekat::units::Units Pa = ekat::units::Pa; ekat::units::Units s = ekat::units::s; ekat::units::Units m = ekat::units::m; @@ -773,22 +769,26 @@ void MAMAci::set_grids( add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field( + // MUST FIXME: remove state_q from this list + /*add_field( "state_q", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars}}, q_unit, - grid_name); // aerosol mmrs [kg/kg] + grid_name); // aerosol mmrs [kg/kg]*/ + // MUST FIXME: Is it same as nc or may be not???? add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] - // add_field("cldo", , unitless, grid_name); // cloud fraction on - // previous time step [fraction] - // + + // MUST FIXME: This should be an internal variable. why we need this as an + // input??? add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] - add_field( + // MUST FIXME: Remove qqcw from here + /*add_field( "qqcw", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios - // [#/kg or kg/kg] + // [#/kg or kg/kg]*/ + // MUST FIXME: move it above so that all "required" variables are together add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] @@ -947,10 +947,6 @@ void MAMAci::set_grids( */ } -size_t MAMAci::requested_buffer_size_in_bytes() const { - return mam_coupling::buffer_size(ncol_, nlev_); -} - void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -968,16 +964,27 @@ void MAMAci::initialize_impl(const RunType run_type) { // set atmosphere state data - p_int_ = get_field_in("p_int").get_view(); - w_sec_ = get_field_in("w_sec").get_view(); - state_q_ = get_field_in("state_q").get_view(); - ncldwtr_ = get_field_in("ncldwtr").get_view(); - qqcw_input_ = get_field_in("qqcw").get_view(); - dgnum_ = get_field_in("dgnum").get_view(); - nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); - niim_ = get_field_out("icenuc_num_immfrz").get_view(); - nidep_ = get_field_out("icenuc_num_depnuc").get_view(); - nimey_ = get_field_out("icenuc_num_meydep").get_view(); + p_int_ = get_field_in("p_int").get_view(); + w_sec_ = get_field_in("w_sec").get_view(); + /*state_q_ = get_field_in("state_q") + .get_view(); // MUST FIXME: remove this*/ + ncldwtr_ = + get_field_in("ncldwtr") + .get_view(); // MUST FIXME: is is nc, may be not??? + /*qqcw_input_ = get_field_in("qqcw") + .get_view(); // MUST FIXME: remove this*/ + dgnum_ = get_field_in("dgnum") + .get_view(); // MUST FIXME: is it an input, can + // we compute it using calcsize??? + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + kvh_ = get_field_in("kvh") + .get_view(); // MUST FIXME: See if scream has it, + // it should com from the land model + + nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); + niim_ = get_field_out("icenuc_num_immfrz").get_view(); + nidep_ = get_field_out("icenuc_num_depnuc").get_view(); + nimey_ = get_field_out("icenuc_num_meydep").get_view(); naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); @@ -992,8 +999,6 @@ void MAMAci::initialize_impl(const RunType run_type) { ccn_ = get_field_out("ccn").get_view(); coltend_outp_ = get_field_out("coltend").get_view(); coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); - kvh_ = get_field_in("kvh").get_view(); wet_atmosphere_.qv = get_field_in("qv").get_view(); wet_atmosphere_.qc = get_field_in("qc").get_view(); @@ -1039,12 +1044,6 @@ void MAMAci::initialize_impl(const RunType run_type) { // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - auto prog_index = [](const int mode, const int species) { - const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); - const int ind = - (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; - return ind; - }; // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = @@ -1062,11 +1061,8 @@ void MAMAci::initialize_impl(const RunType run_type) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - const int index = prog_index(m, a); wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - std::cout << "BALLI:-----" << m << " " << a << " " << index << " " - << wet_aero_.int_aero_mmr[m][a](0, 0) << std::endl; dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } @@ -1074,7 +1070,6 @@ void MAMAci::initialize_impl(const RunType run_type) { const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if(strlen(cld_mmr_field_name) > 0) { - const int index = prog_index(m, a); wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; @@ -1091,21 +1086,21 @@ void MAMAci::initialize_impl(const RunType run_type) { for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. - Kokkos::resize(qqcw_[i], ncol_, nlev_); + /*Kokkos::resize(qqcw_[i], ncol_, nlev_); // MUST FIXME: remove this*/ Kokkos::resize(coltend_[i], ncol_, nlev_); Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { - Kokkos::resize(ptend_q_[i], ncol_, nlev_); + Kokkos::resize(ptend_q_[i], ncol_, nlev_); // MUST FIXME:Do we need this? } for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { + for(int j = 0; j < 2; ++j) { // MUST FIXME:store 2 in a const variable Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); } } - for(int i = 0; i < 42; ++i) + for(int i = 0; i < 42; ++i) // MUST FIXME:store 42 in a const var Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); // nact : fractional aero. number activation rate [/s] @@ -1133,44 +1128,54 @@ void MAMAci::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - // preprocess input -- needs a scan for the calculation of atm height + // preprocess input -- needs a scan for the calculation of local derivied + // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_); + /*copy_scream_array_to_mam4xx( + team_policy, qqcw_, qqcw_input_, + nlev_); // MUST FIXME: remove this and populate qqcw here*/ // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion - compute_w0_and_rho(team_policy, w0_, rho_, wet_atmosphere_, dry_atmosphere_, - top_lev_, nlev_); + compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, + wet_atmosphere_, dry_atmosphere_, top_lev_, nlev_); + + compute_tke_using_w_sec(team_policy, tke_ /*output*/, w_sec_, nlev_); - compute_tke_using_w_sec(team_policy, tke_, w_sec_, nlev_); Kokkos::fence(); // wait for for tke_ to be computed. - compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, + compute_subgrid_scale_velocities(team_policy, wsub_ /*output*/, + wsubice_ /*output*/, wsig_ /*output*/, tke_, wsubmin, top_lev_, nlev_); Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); + compute_subgrid_mean_updraft_velocities(team_policy, w2_ /*output*/, w0_, + wsig_, nlev_); - compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); + compute_aitken_dry_diameter(team_policy, aitken_dry_dia_ /*output*/, dgnum_, + top_lev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + // FIXME: Find out in-outs of the following call! compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, dry_aero_, dry_atmosphere_, aitken_dry_dia_, nlev_); - store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, - wet_atmosphere_, liqcldf_, top_lev_); + store_liquid_cloud_fraction(team_policy, cloud_frac_new_ /*output*/, + cloud_frac_old_ /*output*/, wet_atmosphere_, + liqcldf_, top_lev_); + // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous - // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne - // aerosols, not all the cloud borne aerosols. + // Save cloud borne aerosols to be used in the heterozenous + // freezing before they are changed by the droplet activation + // process. This is only a select subset of cloud borne + // aerosols, not all the cloud borne aerosols. //------------------------------------------------------------- /*NOTE: We probably need to store indices for the select few cloud borne aerosols Fortran code: lchnk_zb = lchnk - begchunk ! save copy of cloud borne @@ -1182,8 +1187,8 @@ void MAMAci::run_impl(const double dt) { rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, - nlev_); + compute_recipical_pseudo_density(team_policy, rpdel_ /*output*/, + dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. #if 0 call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 0523890cff8..5ea30e6ba88 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -62,7 +62,7 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - const_view_3d state_q_; + // const_view_3d state_q_; const_view_2d ncldwtr_; view_2d cloud_frac_new_; @@ -133,15 +133,23 @@ class MAMAci final : public scream::AtmosphereProcess { // Constructor MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // process metadata - AtmosphereProcessType type() const override; - std::string name() const override; + + // Return type of the process + AtmosphereProcessType MAMAci::type() const { + return AtmosphereProcessType::Physics; + } + + // return name of the process + std::string MAMAci::name() const { return "mam4_aci"; } // grid void set_grids( const std::shared_ptr grids_manager) override; // management of common atm process memory - size_t requested_buffer_size_in_bytes() const override; + size_t MAMAci::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); + } void init_buffers(const ATMBufferManager &buffer_manager) override; // process behavior From 6f0132dbe634dcfd1463f073d39a9dfc485ba437 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 14:04:35 -0700 Subject: [PATCH 030/142] Adds call to progs and atm for getting state_q for dropmixnuc --- .../mam/eamxx_mam_aci_process_interface.cpp | 59 +++++++++++-------- .../mam/eamxx_mam_aci_process_interface.hpp | 3 +- .../eamxx/tests/uncoupled/mam4/aci/input.yaml | 2 + 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 0b7fb739e7c..41813a1687b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -373,15 +373,16 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, - mam_coupling::DryAtmosphere &dry_atmosphere, const Real dtmicro, + mam_coupling::DryAtmosphere &dry_atmosphere, + const mam_coupling::AerosolState &dry_aerosol_state, const Real dtmicro, MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], + // MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, MAMAci::const_view_3d state_q, + MAMAci::view_2d rpdel, MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, @@ -420,7 +421,7 @@ void call_function_dropmixnuc( for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; for(int i = 0; i < mam4::ndrop::pver; ++i) for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_qqcw[i] = qqcw[i]; + // for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_qqcw[i] = qqcw[i]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) loc_ptend_q[i] = ptend_q[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; @@ -462,10 +463,10 @@ void call_function_dropmixnuc( raercol_view[i][j] = ekat::subview(loc_raercol[i][j], icol); } } - mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; + /*mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { qqcw_view[i] = ekat::subview(loc_qqcw[i], icol); - } + }*/ mam4::ColumnView ptend_q_view[mam4::ndrop::nvar_ptend_q]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); @@ -491,9 +492,11 @@ void call_function_dropmixnuc( ptend, nctend_mixnuc, factnum) !out */ - // mam4::utils::extract_stateq_from_prognostics(progs, atm, - // state_q_at_kk, - // kk); + mam4::Prognostics prog_at_col = + aerosols_for_column(dry_aerosol_state, icol); + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + /* mam4::ndrop::dropmixnuc( team, dtmicro, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), @@ -525,7 +528,7 @@ void call_function_dropmixnuc( ekat::subview(eddy_diff_km, icol), ekat::subview(qncld, icol), ekat::subview(srcn, icol), ekat::subview(source, icol), ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), - ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)); + ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol));*/ }); } KOKKOS_INLINE_FUNCTION @@ -727,6 +730,8 @@ void MAMAci::set_grids( FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // mid points FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; // interfaces + // layout for 2D (1d horiz X 1d vertical) variable + FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; ekat::units::Units kg = ekat::units::kg; // BQ: why do we need to do this??? ekat::units::Units Pa = ekat::units::Pa; @@ -760,8 +765,9 @@ void MAMAci::set_grids( grid_name); // Total pressure [Pa] at interfaces add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // - // + + add_field("pbl_height", scalar2d_layout_col, m, + grid_name); // planetary boundary layer height add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, @@ -964,7 +970,6 @@ void MAMAci::initialize_impl(const RunType run_type) { // set atmosphere state data - p_int_ = get_field_in("p_int").get_view(); w_sec_ = get_field_in("w_sec").get_view(); /*state_q_ = get_field_in("state_q") .get_view(); // MUST FIXME: remove this*/ @@ -1009,13 +1014,15 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); + dry_atmosphere_.p_int = get_field_in("p_int").get_view(); dry_atmosphere_.p_del = get_field_in("pseudo_density").get_view(); - dry_atmosphere_.qv = buffer_.qv_dry; - dry_atmosphere_.qc = buffer_.qc_dry; - dry_atmosphere_.nc = buffer_.nc_dry; - dry_atmosphere_.qi = buffer_.qi_dry; - dry_atmosphere_.ni = buffer_.ni_dry; + dry_atmosphere_.qv = buffer_.qv_dry; + dry_atmosphere_.qc = buffer_.qc_dry; + dry_atmosphere_.nc = buffer_.nc_dry; + dry_atmosphere_.qi = buffer_.qi_dry; + dry_atmosphere_.ni = buffer_.ni_dry; + dry_atmosphere_.pblh = get_field_in("pbl_height").get_view(); dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above @@ -1087,6 +1094,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. /*Kokkos::resize(qqcw_[i], ncol_, nlev_); // MUST FIXME: remove this*/ + Kokkos::resize(state_q_[i], ncol_, nlev_); Kokkos::resize(coltend_[i], ncol_, nlev_); Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } @@ -1190,14 +1198,15 @@ void MAMAci::run_impl(const double dt) { compute_recipical_pseudo_density(team_policy, rpdel_ /*output*/, dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. -#if 0 - call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, - raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, - p_int_, dry_atmosphere_.p_del, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, - cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); - Kokkos::fence(); // wait for ptend_q_ to be computed. + call_function_dropmixnuc( + team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, + /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, + dry_atmosphere_.p_del, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + Kokkos::fence(); // wait for ptend_q_ to be computed. +#if 0 copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, nlev_); copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, nlev_); copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 5ea30e6ba88..e22e1f54e6f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -62,7 +62,7 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - // const_view_3d state_q_; + view_2d state_q_[mam4::ndrop::ncnst_tot]; const_view_2d ncldwtr_; view_2d cloud_frac_new_; @@ -112,7 +112,6 @@ class MAMAci final : public scream::AtmosphereProcess { const int top_lev_ = 6; // local atmospheric state column variables - const_view_2d p_int_; // Total pressure [Pa] at interfaces const_view_2d pdel_; // pressure thickess of layer [Pa] view_2d rpdel_; // Inverse of pdel_ const_view_2d w_sec_; // Vertical velocity variance diff --git a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml index dc807241f96..946d11dcc0e 100644 --- a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml @@ -26,10 +26,12 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} T_mid: 273.0 p_mid: 1.e5 + p_int: 1.e5 num_a1: 1.e5 num_c1: 1.e5 bc_a1: 1.e-5 qqcw: 1e5 + pbl_height: 0.0 aero_gas_mmr_o3: 0.0 aero_gas_mmr_h2o2: 0.0 From 5f5f2a765499c45612f4cdc953f9bfb3db7985e4 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 17:01:44 -0700 Subject: [PATCH 031/142] Adds a working state_q extraction --- .../mam/eamxx_mam_aci_process_interface.cpp | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 41813a1687b..e6b2ca1ac7f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -382,7 +382,7 @@ void call_function_dropmixnuc( MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot], + MAMAci::view_2d rpdel, /*MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot],*/ MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, @@ -428,6 +428,11 @@ void call_function_dropmixnuc( for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend_cw[i] = coltend_cw[i]; + MAMAci::view_1d state_q[mam4::ndrop::ncnst_tot]; + + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + Kokkos::resize(state_q[i], mam4::ndrop::pver); + } Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -492,10 +497,22 @@ void call_function_dropmixnuc( ptend, nctend_mixnuc, factnum) !out */ - mam4::Prognostics prog_at_col = + mam4::Prognostics progs_at_col = aerosols_for_column(dry_aerosol_state, icol); haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); + + for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { + Real state_q_at_lev_col[mam4::ndrop::ncnst_tot] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + // mam4::utils::extract_stateq_from_prognostics(progs_at_col, + // haero_atm, state_q, klev); + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + state_q[i](klev) = state_q_at_lev_col[i]; + } + } + /* mam4::ndrop::dropmixnuc( team, dtmicro, ekat::subview(T_mid, icol), @@ -1202,7 +1219,7 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc( team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, - dry_atmosphere_.p_del, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + dry_atmosphere_.p_del, rpdel_, /*state_q_,*/ ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. From 96f577bf3ea43fbe818cf3f3c1b00b77d29d2b97 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 17:13:21 -0700 Subject: [PATCH 032/142] Adds a working qqcw for dropmixnuc --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e6b2ca1ac7f..99d1a92e4ae 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -429,9 +429,11 @@ void call_function_dropmixnuc( loc_coltend_cw[i] = coltend_cw[i]; MAMAci::view_1d state_q[mam4::ndrop::ncnst_tot]; + MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { Kokkos::resize(state_q[i], mam4::ndrop::pver); + Kokkos::resize(qqcw[i], mam4::ndrop::pver); } Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -504,12 +506,15 @@ void call_function_dropmixnuc( for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { Real state_q_at_lev_col[mam4::ndrop::ncnst_tot] = {}; + Real qqcw_at_lev_col[mam4::ndrop::ncnst_tot] = {}; mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); - // mam4::utils::extract_stateq_from_prognostics(progs_at_col, - // haero_atm, state_q, klev); + + mam4::utils::extract_qqcw_from_prognostics(progs_at_col, + qqcw_at_lev_col, klev); for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { state_q[i](klev) = state_q_at_lev_col[i]; + qqcw[i](klev) = qqcw_at_lev_col[i]; } } From ce90fd2d198264cae3239d33656d23e63b2a020e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 17:35:21 -0700 Subject: [PATCH 033/142] A working state_q1 as a 2d view --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 99d1a92e4ae..91ada8a87dd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -429,6 +429,8 @@ void call_function_dropmixnuc( loc_coltend_cw[i] = coltend_cw[i]; MAMAci::view_1d state_q[mam4::ndrop::ncnst_tot]; + MAMAci::view_2d state_q1; + Kokkos::resize(state_q1, mam4::ndrop::pver, mam4::ndrop::ncnst_tot); MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { @@ -513,8 +515,9 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - state_q[i](klev) = state_q_at_lev_col[i]; - qqcw[i](klev) = qqcw_at_lev_col[i]; + state_q[i](klev) = state_q_at_lev_col[i]; + state_q1(klev, i) = state_q_at_lev_col[i]; + qqcw[i](klev) = qqcw_at_lev_col[i]; } } From 5c5f90bb10acf1589df2ec52a777b0fc0237062f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 20:08:26 -0700 Subject: [PATCH 034/142] Particle args for dropmixnuc upto state_q works --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 91ada8a87dd..ac313629318 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -521,7 +521,6 @@ void call_function_dropmixnuc( } } - /* mam4::ndrop::dropmixnuc( team, dtmicro, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), @@ -529,7 +528,7 @@ void call_function_dropmixnuc( ekat::subview( zm, icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - ekat::subview(state_q, icol), ekat::subview(ncldwtr, icol), + state_q1/*, ekat::subview(ncldwtr, icol), ekat::subview(kvh, icol), // kvh[kk+1] ekat::subview(cloud_frac_new, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, @@ -553,7 +552,7 @@ void call_function_dropmixnuc( ekat::subview(eddy_diff_km, icol), ekat::subview(qncld, icol), ekat::subview(srcn, icol), ekat::subview(source, icol), ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), - ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol));*/ + ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)*/); }); } KOKKOS_INLINE_FUNCTION From 6fd9beca3d599a69769d1a95f19f5cd081d6b8ec Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 18 Mar 2024 14:08:39 -0700 Subject: [PATCH 035/142] A working qqcw using a scratch memory with partial arg list --- .../mam/eamxx_mam_aci_process_interface.cpp | 62 +++++++++++++------ .../mam/eamxx_mam_aci_process_interface.hpp | 3 + 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ac313629318..91dce126286 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -378,6 +378,7 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], // MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], + MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], @@ -428,18 +429,29 @@ void call_function_dropmixnuc( for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend_cw[i] = coltend_cw[i]; - MAMAci::view_1d state_q[mam4::ndrop::ncnst_tot]; - MAMAci::view_2d state_q1; - Kokkos::resize(state_q1, mam4::ndrop::pver, mam4::ndrop::ncnst_tot); - MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; + MAMAci::view_2d state_q; + Kokkos::resize( + state_q, mam4::ndrop::pver, + mam4::ndrop::nvars); // FIXME: use pcnst here and add ncol dim as well + + // mam4::ColumnView qqcw[mam4::ndrop::ncnst_tot]; + // qqcw = work_; + // MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; + + // for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + // Kokkos::resize( + + /*MAMAci::view_2d &qqcw_ptr[25]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + qqcw_ptr[i] = qqcw_fld_work_[i]*/ + MAMAci::view_2d qqcw_fld_work_loc[25]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - Kokkos::resize(state_q[i], mam4::ndrop::pver); - Kokkos::resize(qqcw[i], mam4::ndrop::pver); - } Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); + // for (int icol =0 ; icol<1 ;icol++){ // Initialize the ndrop class. const int ntot_amode = mam_coupling::num_aero_modes(); @@ -472,10 +484,12 @@ void call_function_dropmixnuc( raercol_view[i][j] = ekat::subview(loc_raercol[i][j], icol); } } - /*mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; + mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - qqcw_view[i] = ekat::subview(loc_qqcw[i], icol); - }*/ + // qqcw_view[i] = ekat::subview(loc_qqcw[i], icol); + qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); + } + mam4::ColumnView ptend_q_view[mam4::ndrop::nvar_ptend_q]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); @@ -507,17 +521,19 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::ndrop::ncnst_tot] = {}; - Real qqcw_at_lev_col[mam4::ndrop::ncnst_tot] = {}; + Real state_q_at_lev_col[mam4::ndrop::nvars] = {}; // use pcnst here + Real qqcw_at_lev_col[mam4::ndrop::nvars] = {}; // use pcnst here mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - state_q[i](klev) = state_q_at_lev_col[i]; - state_q1(klev, i) = state_q_at_lev_col[i]; - qqcw[i](klev) = qqcw_at_lev_col[i]; + for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { + state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; + // std::cout<<"BALLI:"<(); + // allocate work + + for(int icnst = 0; icnst < 25; ++icnst) { + qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); + } + // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -1225,6 +1248,7 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc( team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, + qqcw_fld_work_, /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, dry_atmosphere_.p_del, rpdel_, /*state_q_,*/ ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index e22e1f54e6f..79f099b7d43 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -128,6 +128,9 @@ class MAMAci final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; + // A view array to carry cloud borne aerosol mmrs/nmrs + view_2d qqcw_fld_work_[25]; + public: // Constructor MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); From 812bc3f28da2800fa5ec6e95eac13a70b880a6a7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 18 Mar 2024 14:14:11 -0700 Subject: [PATCH 036/142] Full dropmixnuc call is working now --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 91dce126286..ae41bba0723 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -529,10 +529,7 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { - state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; - // std::cout<<"BALLI:"< Date: Mon, 18 Mar 2024 14:40:53 -0700 Subject: [PATCH 037/142] Creates state_q with columns and scratch memory --- .../mam/eamxx_mam_aci_process_interface.cpp | 47 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 + 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ae41bba0723..8ab58132e57 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -384,12 +384,12 @@ void call_function_dropmixnuc( MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, MAMAci::view_2d rpdel, /*MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot],*/ - MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, - MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, - MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, - MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_3d state_q_work_, MAMAci::const_view_2d ncldwtr, + MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, MAMAci::view_2d wsub, + MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, + MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, + MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, + MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -434,20 +434,12 @@ void call_function_dropmixnuc( state_q, mam4::ndrop::pver, mam4::ndrop::nvars); // FIXME: use pcnst here and add ncol dim as well - // mam4::ColumnView qqcw[mam4::ndrop::ncnst_tot]; - // qqcw = work_; - // MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; - - // for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) - // Kokkos::resize( - - /*MAMAci::view_2d &qqcw_ptr[25]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) - qqcw_ptr[i] = qqcw_fld_work_[i]*/ MAMAci::view_2d qqcw_fld_work_loc[25]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; + MAMAci::view_3d state_q_work_loc = state_q_work_; + Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -529,8 +521,9 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { - state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; - qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; + state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; } } @@ -541,7 +534,7 @@ void call_function_dropmixnuc( ekat::subview( zm, icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - state_q, ekat::subview(ncldwtr, icol), + ekat::subview(state_q_work_loc, icol), ekat::subview(ncldwtr, icol), ekat::subview(kvh, icol), // kvh[kk+1] ekat::subview(cloud_frac_new, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, @@ -1091,6 +1084,7 @@ void MAMAci::initialize_impl(const RunType run_type) { for(int icnst = 0; icnst < 25; ++icnst) { qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); } + state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, 40); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -1243,13 +1237,14 @@ void MAMAci::run_impl(const double dt) { dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. - call_function_dropmixnuc( - team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, - qqcw_fld_work_, - /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, - dry_atmosphere_.p_del, rpdel_, /*state_q_,*/ ncldwtr_, kvh_, qcld_, wsub_, - cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + call_function_dropmixnuc(team_policy, dry_atmosphere_, dry_aero_, dtmicro_, + raercol_cw_, raercol_, qqcw_fld_work_, + /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, + dry_atmosphere_.p_int, dry_atmosphere_.p_del, rpdel_, + /*state_q_,*/ state_q_work_, ncldwtr_, kvh_, qcld_, + wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, + factnum_, ndropcol_, ndropmix_, nsource_, wtke_, + ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. #if 0 copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 79f099b7d43..b6e52026801 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -130,6 +130,7 @@ class MAMAci final : public scream::AtmosphereProcess { // A view array to carry cloud borne aerosol mmrs/nmrs view_2d qqcw_fld_work_[25]; + view_3d state_q_work_; public: // Constructor From a215ef3f27195b3fbdc49163e71ea01ad884b1bc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 18 Mar 2024 17:57:27 -0700 Subject: [PATCH 038/142] Some cleanup-still need a lot of cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 64 +++++-------------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 - 2 files changed, 15 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 8ab58132e57..dd415e970c6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -377,19 +377,18 @@ void call_function_dropmixnuc( const mam_coupling::AerosolState &dry_aerosol_state, const Real dtmicro, MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], - // MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, /*MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot],*/ - MAMAci::view_3d state_q_work_, MAMAci::const_view_2d ncldwtr, - MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, MAMAci::view_2d wsub, - MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, - MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, - MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, - MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_2d rpdel, MAMAci::view_3d state_q_work_, + MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, + MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, + MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, + MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, + MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -422,18 +421,12 @@ void call_function_dropmixnuc( for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; for(int i = 0; i < mam4::ndrop::pver; ++i) for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - // for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_qqcw[i] = qqcw[i]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) loc_ptend_q[i] = ptend_q[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend_cw[i] = coltend_cw[i]; - MAMAci::view_2d state_q; - Kokkos::resize( - state_q, mam4::ndrop::pver, - mam4::ndrop::nvars); // FIXME: use pcnst here and add ncol dim as well - MAMAci::view_2d qqcw_fld_work_loc[25]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; @@ -443,7 +436,6 @@ void call_function_dropmixnuc( Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // for (int icol =0 ; icol<1 ;icol++){ // Initialize the ndrop class. const int ntot_amode = mam_coupling::num_aero_modes(); @@ -478,10 +470,8 @@ void call_function_dropmixnuc( } mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - // qqcw_view[i] = ekat::subview(loc_qqcw[i], icol); qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); } - mam4::ColumnView ptend_q_view[mam4::ndrop::nvar_ptend_q]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); @@ -521,7 +511,6 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { - state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; } @@ -560,7 +549,6 @@ void call_function_dropmixnuc( ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)); }); - // } } KOKKOS_INLINE_FUNCTION void copy_mam4xx_array_to_scream(const haero::ThreadTeam &team, @@ -806,11 +794,6 @@ void MAMAci::set_grids( add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // MUST FIXME: remove state_q from this list - /*add_field( - "state_q", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars}}, q_unit, - grid_name); // aerosol mmrs [kg/kg]*/ // MUST FIXME: Is it same as nc or may be not???? add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] @@ -818,13 +801,7 @@ void MAMAci::set_grids( // MUST FIXME: This should be an internal variable. why we need this as an // input??? add_field("w_updraft", scalar3d_layout_mid, q_unit, - grid_name); // updraft velocity [m/s] - // MUST FIXME: Remove qqcw from here - /*add_field( - "qqcw", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios - // [#/kg or kg/kg]*/ + grid_name); // updraft velocity // MUST FIXME: move it above so that all "required" variables are together add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] @@ -1002,13 +979,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // set atmosphere state data w_sec_ = get_field_in("w_sec").get_view(); - /*state_q_ = get_field_in("state_q") - .get_view(); // MUST FIXME: remove this*/ ncldwtr_ = get_field_in("ncldwtr") .get_view(); // MUST FIXME: is is nc, may be not??? - /*qqcw_input_ = get_field_in("qqcw") - .get_view(); // MUST FIXME: remove this*/ dgnum_ = get_field_in("dgnum") .get_view(); // MUST FIXME: is it an input, can // we compute it using calcsize??? @@ -1131,8 +1104,6 @@ void MAMAci::initialize_impl(const RunType run_type) { for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. - /*Kokkos::resize(qqcw_[i], ncol_, nlev_); // MUST FIXME: remove this*/ - Kokkos::resize(state_q_[i], ncol_, nlev_); Kokkos::resize(coltend_[i], ncol_, nlev_); Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } @@ -1181,10 +1152,6 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - /*copy_scream_array_to_mam4xx( - team_policy, qqcw_, qqcw_input_, - nlev_); // MUST FIXME: remove this and populate qqcw here*/ - // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion @@ -1237,14 +1204,13 @@ void MAMAci::run_impl(const double dt) { dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. - call_function_dropmixnuc(team_policy, dry_atmosphere_, dry_aero_, dtmicro_, - raercol_cw_, raercol_, qqcw_fld_work_, - /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, - dry_atmosphere_.p_int, dry_atmosphere_.p_del, rpdel_, - /*state_q_,*/ state_q_work_, ncldwtr_, kvh_, qcld_, - wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, - factnum_, ndropcol_, ndropmix_, nsource_, wtke_, - ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + call_function_dropmixnuc( + team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, + qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, + dry_atmosphere_.p_del, rpdel_, state_q_work_, ncldwtr_, kvh_, qcld_, + wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, + ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, + nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. #if 0 copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index b6e52026801..88d91b999cd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -62,7 +62,6 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - view_2d state_q_[mam4::ndrop::ncnst_tot]; const_view_2d ncldwtr_; view_2d cloud_frac_new_; From fb40cd6dcbae8bc893698e9f58aa14ec6afb67fb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 18 Mar 2024 17:59:05 -0700 Subject: [PATCH 039/142] Adds a comment --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index dd415e970c6..776a0acfdf4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -511,8 +511,10 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; + state_q_work_loc(icol, klev, icnst) = + state_q_at_lev_col[icnst]; // FIXME: ensure that indices are + // right! remove "15" if possible!! + qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; } } From 7a16e65b38a70a3bcd1f8e7eb361b0fb1816a2eb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 19 Mar 2024 21:54:13 -0700 Subject: [PATCH 040/142] Modified species order to match E3SM with the folling change in mam4xx mam4xx/aero_model.hpp /// Identifiers for aerosol species that inhabit MAM4 modes. enum class AeroId { SOA = 2, // secondary organic aerosol SO4 = 0, // sulphate POM = 1, // primary organic matter BC = 3, // black carbon NaCl = 5, // sodium chloride DST = 4, // dust MOM = 6, // marine organic matter, None = 7 // invalid aerosol species }; --- .../mam/eamxx_mam_aci_process_interface.cpp | 245 ++--- .../eamxx/src/physics/mam/mam_coupling.hpp | 861 +++++++++--------- .../eamxx/tests/uncoupled/mam4/aci/input.yaml | 7 +- 3 files changed, 571 insertions(+), 542 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 776a0acfdf4..ee8fb3b130e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -602,111 +602,112 @@ void call_hetfrz_compute_tendencies( mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; - ; MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d nc = wet_atmosphere.nc; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies - // class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, - ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, - dummy, dummy, dummy, dummy, dummy, pblh); - // set surface state data - haero::Surface surf{}; - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); - - const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - - mam4::Diagnostics diags(nlev); - diags.stratiform_cloud_fraction = - ekat::subview(stratiform_cloud_fraction, icol); - diags.activation_fraction[accum_idx] = - ekat::subview(activation_fraction_accum_idx, icol); - diags.activation_fraction[coarse_idx] = - ekat::subview(activation_fraction_coarse_idx, icol); - - // These are the output tendencies from heterogeneous freezing that need - // to be added correctly to the cloud-micorphysics scheme. - diags.hetfrz_immersion_nucleation_tend = - ekat::subview(hetfrz_immersion_nucleation_tend, icol); - diags.hetfrz_contact_nucleation_tend = - ekat::subview(hetfrz_contact_nucleation_tend, icol); - diags.hetfrz_depostion_nucleation_tend = - ekat::subview(hetfrz_depostion_nucleation_tend, icol); - - diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); - diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); - diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); - diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); - diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); - diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); - diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); - diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); - diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); - diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); - diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); - diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); - diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); - diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); - diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); - diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); - diags.na500 = ekat::subview(diagnostic_scratch[18], icol); - diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); - diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); - diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); - diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); - diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); - diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); - diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); - diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); - diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); - diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); - diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); - diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); - diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); - diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); - diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); - diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); - diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); - diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); - diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); - diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); - diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); - diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - //------------------------------------------------------------- - // Heterogeneous freezing - // frzimm, frzcnt, frzdep are the outputs of - // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) - //------------------------------------------------------------- - // - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t = 0, dt = 0; - hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, - diags, tends); - }); + // Kokkos::parallel_for( + // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + // const int icol = team.league_rank(); + for(int icol = 0; icol < 1; icol++) { + // Set up an atmosphere, surface, diagnostics, pronostics and tendencies + // class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, + ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, dummy, + dummy, dummy, dummy, dummy, pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + + // BALLI + mam4::Diagnostics diags(nlev); + diags.stratiform_cloud_fraction = + ekat::subview(stratiform_cloud_fraction, icol); + diags.activation_fraction[accum_idx] = + ekat::subview(activation_fraction_accum_idx, icol); + diags.activation_fraction[coarse_idx] = + ekat::subview(activation_fraction_coarse_idx, icol); + + // These are the output tendencies from heterogeneous freezing that need + // to be added correctly to the cloud-micorphysics scheme. + diags.hetfrz_immersion_nucleation_tend = + ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = + ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = + ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0, dt = 0; + hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, progs, + diags, tends); + //}); + } } } // namespace @@ -880,6 +881,7 @@ void MAMAci::set_grids( mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + std::cout << "mode:" << mode << " name:" << int_nmr_field_name << std::endl; // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -888,23 +890,30 @@ void MAMAci::set_grids( mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); + // std::cout<<"mode:"<< <<" name:"<< < 0) + if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - + std::cout << "--> mode:" << mode << " spec:" << a + << " int-spec-name:" << int_mmr_field_name << std::endl; + } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); - if(strlen(cld_mmr_field_name) > 0) + if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); + // std::cout<<"--> mode:"<< mode<<" spec:"<< a<< " cld-spec name:"<< + // cld_mmr_field_name<(team_policy, ptend_q_output_, ptend_q_, nlev_); - copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, nlev_); - copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - - call_hetfrz_compute_tendencies(team_policy, - hetfrz_, dry_aero_, wet_atmosphere_, dry_atmosphere_, - stratiform_cloud_fraction_, - activation_fraction_accum_idx_, activation_fraction_coarse_idx_, - hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, - naai_hom_, naai_, - diagnostic_scratch_, - nlev_); - - Kokkos::fence(); // wait before returning to calling function -#endif + + copy_mam4xx_array_to_scream( + team_policy, ptend_q_output_, ptend_q_, nlev_); + copy_mam4xx_array_to_scream( + team_policy, coltend_outp_, coltend_, nlev_); + copy_mam4xx_array_to_scream( + team_policy, coltend_cw_outp_, coltend_cw_, nlev_); + + call_hetfrz_compute_tendencies( + team_policy, hetfrz_, dry_aero_, wet_atmosphere_, dry_atmosphere_, + stratiform_cloud_fraction_, activation_fraction_accum_idx_, + activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, + hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, + naai_hom_, naai_, diagnostic_scratch_, nlev_); + + Kokkos::fence(); // wait before returning to calling function } void MAMAci::finalize_impl() { diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index c864a727550..9cc487dbe28 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -1,9 +1,9 @@ #ifndef MAM_COUPLING_HPP #define MAM_COUPLING_HPP -#include -#include #include +#include +#include #include #include @@ -13,7 +13,7 @@ namespace scream::mam_coupling { -using KT = ekat::KokkosTypes; +using KT = ekat::KokkosTypes; // views for single- and multi-column data using view_1d = typename KT::template view_1d; @@ -27,8 +27,10 @@ using const_view_3d = typename KT::template view_3d; using Team = Kokkos::TeamPolicy::member_type; // unmanaged views (for buffer and workspace manager) -using uview_1d = typename ekat::template Unmanaged>; -using uview_2d = typename ekat::template Unmanaged>; +using uview_1d = + typename ekat::template Unmanaged>; +using uview_2d = + typename ekat::template Unmanaged>; using PF = scream::PhysicsFunctions; @@ -45,21 +47,15 @@ constexpr int nqtendbb() { return 4; } // returns the number of distinct aerosol modes KOKKOS_INLINE_FUNCTION -constexpr int num_aero_modes() { - return mam4::AeroConfig::num_modes(); -} +constexpr int num_aero_modes() { return mam4::AeroConfig::num_modes(); } // returns the number of distinct aerosol species KOKKOS_INLINE_FUNCTION -constexpr int num_aero_species() { - return mam4::AeroConfig::num_aerosol_ids(); -} +constexpr int num_aero_species() { return mam4::AeroConfig::num_aerosol_ids(); } // returns the number of distinct aerosol-related gases KOKKOS_INLINE_FUNCTION -constexpr int num_aero_gases() { - return mam4::AeroConfig::num_gas_ids(); -} +constexpr int num_aero_gases() { return mam4::AeroConfig::num_gas_ids(); } // returns the total number of aerosol tracers (i.e. the total number of // distinct valid mode-species pairs) @@ -73,12 +69,12 @@ constexpr int num_aero_tracers() { // Given a MAM aerosol mode index, returns a string denoting the symbolic // name of the mode. KOKKOS_INLINE_FUNCTION -const char* aero_mode_name(const int mode) { +const char *aero_mode_name(const int mode) { static const char *mode_names[num_aero_modes()] = { - "1", - "2", - "3", - "4", + "1", + "2", + "3", + "4", }; return mode_names[mode]; } @@ -86,31 +82,18 @@ const char* aero_mode_name(const int mode) { // Given a MAM aerosol species ID, returns a string denoting the symbolic // name of the species. KOKKOS_INLINE_FUNCTION -const char* aero_species_name(const int species_id) { +const char *aero_species_name(const int species_id) { static const char *species_names[num_aero_species()] = { - "soa", - "so4", - "pom", - "bc", - "nacl", - "dst", - "mom", - }; + "so4", "pom", "soa", "bc", "dst", "nacl", "mom"}; return species_names[species_id]; } // Given a MAM aerosol-related gas ID, returns a string denoting the symbolic // name of the gas species. KOKKOS_INLINE_FUNCTION -const char* gas_species_name(const int gas_id) { - static const char *species_names[num_aero_gases()] = { - "o3", - "h2o2", - "h2so4", - "so2", - "dms", - "soag" - }; +const char *gas_species_name(const int gas_id) { + static const char *species_names[num_aero_gases()] = {"o3", "h2o2", "h2so4", + "so2", "dms", "soag"}; return species_names[gas_id]; } @@ -118,154 +101,155 @@ const char* gas_species_name(const int gas_id) { namespace { KOKKOS_INLINE_FUNCTION -constexpr int max_field_name_len() { - return 128; -} +constexpr int max_field_name_len() { return 128; } KOKKOS_INLINE_FUNCTION -size_t gpu_strlen(const char* s) { +size_t gpu_strlen(const char *s) { size_t l = 0; - while (s[l]) ++l; + while(s[l]) ++l; return l; } KOKKOS_INLINE_FUNCTION void concat_2_strings(const char *s1, const char *s2, char *concatted) { size_t len1 = gpu_strlen(s1); - for (size_t i = 0; i < len1; ++i) - concatted[i] = s1[i]; + for(size_t i = 0; i < len1; ++i) concatted[i] = s1[i]; size_t len2 = gpu_strlen(s2); - for (size_t i = 0; i < len2; ++i) - concatted[i + len1] = s2[i]; - concatted[len1+len2] = 0; + for(size_t i = 0; i < len2; ++i) concatted[i + len1] = s2[i]; + concatted[len1 + len2] = 0; } KOKKOS_INLINE_FUNCTION -void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { +void concat_3_strings(const char *s1, const char *s2, const char *s3, + char *concatted) { size_t len1 = gpu_strlen(s1); - for (size_t i = 0; i < len1; ++i) - concatted[i] = s1[i]; + for(size_t i = 0; i < len1; ++i) concatted[i] = s1[i]; size_t len2 = gpu_strlen(s2); - for (size_t i = 0; i < len2; ++i) - concatted[i + len1] = s2[i]; + for(size_t i = 0; i < len2; ++i) concatted[i + len1] = s2[i]; size_t len3 = gpu_strlen(s3); - for (size_t i = 0; i < len3; ++i) - concatted[i + len1 + len2] = s3[i]; - concatted[len1+len2+len3] = 0; + for(size_t i = 0; i < len3; ++i) concatted[i + len1 + len2] = s3[i]; + concatted[len1 + len2 + len3] = 0; } KOKKOS_INLINE_FUNCTION -char* int_aero_nmr_names(int mode) { +char *int_aero_nmr_names(int mode) { static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return int_aero_nmr_names_[mode]; } KOKKOS_INLINE_FUNCTION -char* cld_aero_nmr_names(int mode) { +char *cld_aero_nmr_names(int mode) { static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return cld_aero_nmr_names_[mode]; } KOKKOS_INLINE_FUNCTION -char* int_aero_mmr_names(int mode, int species) { - static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; +char *int_aero_mmr_names(int mode, int species) { + static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()] + [max_field_name_len()] = {}; return int_aero_mmr_names_[mode][species]; } KOKKOS_INLINE_FUNCTION -char* cld_aero_mmr_names(int mode, int species) { - static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; +char *cld_aero_mmr_names(int mode, int species) { + static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()] + [max_field_name_len()] = {}; return cld_aero_mmr_names_[mode][species]; } KOKKOS_INLINE_FUNCTION -char* gas_mmr_names(int gas_id) { +char *gas_mmr_names(int gas_id) { static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; return gas_mmr_names_[gas_id]; } -} // end anonymous namespace +} // end anonymous namespace // Given a MAM aerosol mode index, returns the name of the related interstitial // modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") KOKKOS_INLINE_FUNCTION -const char* int_aero_nmr_field_name(const int mode) { - if (!int_aero_nmr_names(mode)[0]) { +const char *int_aero_nmr_field_name(const int mode) { + if(!int_aero_nmr_names(mode)[0]) { concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); } - return const_cast(int_aero_nmr_names(mode)); + return const_cast(int_aero_nmr_names(mode)); } // Given a MAM aerosol mode index, returns the name of the related cloudborne // modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") KOKKOS_INLINE_FUNCTION -const char* cld_aero_nmr_field_name(const int mode) { - if (!cld_aero_nmr_names(mode)[0]) { +const char *cld_aero_nmr_field_name(const int mode) { + if(!cld_aero_nmr_names(mode)[0]) { concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); } - return const_cast(cld_aero_nmr_names(mode)); + return const_cast(cld_aero_nmr_names(mode)); } // Given a MAM aerosol mode index and the index of the MAM aerosol species // within it, returns the name of the relevant interstitial mass mixing ratio -// field in EAMxx. The form of the field name is "_a<1-based-mode-index>". -// If the desired species is not present within the desire mode, returns a blank -// string (""). +// field in EAMxx. The form of the field name is +// "_a<1-based-mode-index>". If the desired species is not present +// within the desire mode, returns a blank string (""). KOKKOS_INLINE_FUNCTION -const char* int_aero_mmr_field_name(const int mode, const int species) { - if (!int_aero_mmr_names(mode, species)[0]) { +const char *int_aero_mmr_field_name(const int mode, const int species) { + if(!int_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); - if (aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), - "_a", aero_mode_name(mode), - int_aero_mmr_names(mode, species)); + if(aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), "_a", + aero_mode_name(mode), int_aero_mmr_names(mode, species)); } } - return const_cast(int_aero_mmr_names(mode, species)); + return const_cast(int_aero_mmr_names(mode, species)); }; // Given a MAM aerosol mode index and the index of the MAM aerosol species // within it, returns the name of the relevant cloudborne mass mixing ratio -// field in EAMxx. The form of the field name is "_c<1-based-mode-index>". -// If the desired species is not present within the desire mode, returns a blank -// string (""). +// field in EAMxx. The form of the field name is +// "_c<1-based-mode-index>". If the desired species is not present +// within the desire mode, returns a blank string (""). KOKKOS_INLINE_FUNCTION -const char* cld_aero_mmr_field_name(const int mode, const int species) { - if (!cld_aero_mmr_names(mode, species)[0]) { +const char *cld_aero_mmr_field_name(const int mode, const int species) { + if(!cld_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); - if (aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), - "_c", aero_mode_name(mode), - cld_aero_mmr_names(mode, species)); + if(aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), "_c", + aero_mode_name(mode), cld_aero_mmr_names(mode, species)); } } - return const_cast(cld_aero_mmr_names(mode, species)); + return const_cast(cld_aero_mmr_names(mode, species)); }; // Given a MAM aerosol-related gas identifier, returns the name of its mass // mixing ratio field in EAMxx ("aero_gas_mmr_") KOKKOS_INLINE_FUNCTION -const char* gas_mmr_field_name(const int gas) { - if (!gas_mmr_names(gas)[0]) { - concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); +const char *gas_mmr_field_name(const int gas) { + if(!gas_mmr_names(gas)[0]) { + concat_2_strings("aero_gas_mmr_", gas_species_name(gas), + gas_mmr_names(gas)); } - return const_cast(gas_mmr_names(gas)); + return const_cast(gas_mmr_names(gas)); } // This type stores multi-column views related specifically to the wet // atmospheric state used by EAMxx. struct WetAtmosphere { - const_view_2d qv; // wet water vapor specific humidity [kg vapor / kg moist air] - const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud water/kg moist air] - const_view_2d nc; // wet cloud liquid water number mixing ratio [# / kg moist air] - const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice water / kg moist air] - const_view_2d ni; // wet cloud ice water number mixing ratio [# / kg moist air] - const_view_2d omega; // vertical pressure velocity [Pa/s] + const_view_2d + qv; // wet water vapor specific humidity [kg vapor / kg moist air] + const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud + // water/kg moist air] + const_view_2d + nc; // wet cloud liquid water number mixing ratio [# / kg moist air] + const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice + // water / kg moist air] + const_view_2d + ni; // wet cloud ice water number mixing ratio [# / kg moist air] + const_view_2d omega; // vertical pressure velocity [Pa/s] }; // This type stores multi-column views related to the dry atmospheric state // used by MAM. struct DryAtmosphere { +<<<<<<< HEAD Real z_surf; // height of bottom of atmosphere [m] const_view_2d T_mid; // temperature at grid midpoints [K] const_view_2d p_mid; // total pressure at grid midpoints [Pa] @@ -283,20 +267,50 @@ struct DryAtmosphere { view_2d w_updraft; // updraft velocity [m/s] const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] +======= + Real z_surf; // height of bottom of atmosphere [m] + const_view_2d T_mid; // temperature at grid midpoints [K] + const_view_2d p_mid; // total pressure at grid midpoints [Pa] + view_2d qv; // dry water vapor mixing ratio [kg vapor / kg dry air] + view_2d qc; // dry cloud liquid water mass mixing ratio [kg cloud water/kg + // dry air] + view_2d nc; // dry cloud liquid water number mixing ratio [# / kg dry air] + view_2d qi; // dry cloud ice water mass mixing ratio [kg cloud ice water / kg + // dry air] + view_2d ni; // dry cloud ice water number mixing ratio [# / kg dry air] + view_2d z_mid; // height at layer midpoints [m] + view_2d z_iface; // height at layer interfaces [m] + view_2d dz; // layer thickness [m] + const_view_2d + p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] + const_view_2d p_int; // total pressure at grid interfaces [Pa] + const_view_2d cldfrac; // cloud fraction [-] + view_2d w_updraft; // updraft velocity [m/s] + const_view_1d pblh; // planetary boundary layer height [m] + const_view_1d phis; // surface geopotential [m2/s2] +>>>>>>> Modified species order to match E3SM with the folling change in mam4xx }; // This type stores aerosol number and mass mixing ratios evolved by MAM. It // can be used to represent wet and dry aerosols. When you declare an // AerosolState, you must decide whether it's a dry or wet aerosol state (with // mixing ratios in terms of dry or wet parcels of air, respectively). -// These mixing ratios are organized by mode (and species, for mass mixing ratio) -// in the same way as they are in mam4xx, and indexed using mam4::AeroConfig. +// These mixing ratios are organized by mode (and species, for mass mixing +// ratio) in the same way as they are in mam4xx, and indexed using +// mam4::AeroConfig. struct AerosolState { - view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number mixing ratios [# / kg air] - view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number mixing ratios [# / kg air] - view_2d int_aero_mmr[num_aero_modes()][num_aero_species()]; // interstitial aerosol mass mixing ratios [kg aerosol / kg air] - view_2d cld_aero_mmr[num_aero_modes()][num_aero_species()]; // cloudborne aerosol mass mixing ratios [kg aerosol / kg air] - view_2d gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] + view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number + // mixing ratios [# / kg air] + view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number + // mixing ratios [# / kg air] + view_2d int_aero_mmr[num_aero_modes()] + [num_aero_species()]; // interstitial aerosol mass mixing + // ratios [kg aerosol / kg air] + view_2d cld_aero_mmr[num_aero_modes()] + [num_aero_species()]; // cloudborne aerosol mass mixing + // ratios [kg aerosol / kg air] + view_2d + gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] }; // storage for variables used within MAM atmosphere processes, initialized with @@ -312,20 +326,20 @@ struct Buffer { static constexpr int num_2d_scratch = 10; // number of local fields stored at column midpoints - static constexpr int num_2d_mid = 8 + // number of dry atm fields - 2 * (num_aero_modes() + num_aero_tracers()) + - num_aero_gases() + - num_2d_scratch; + static constexpr int num_2d_mid = + 8 + // number of dry atm fields + 2 * (num_aero_modes() + num_aero_tracers()) + num_aero_gases() + + num_2d_scratch; // (dry) atmospheric state - uview_2d z_mid; // height at midpoints - uview_2d dz; // layer thickness - uview_2d qv_dry; // dry water vapor mixing ratio (dry air) - uview_2d qc_dry; // dry cloud water mass mixing ratio - uview_2d nc_dry; // dry cloud water number mixing ratio - uview_2d qi_dry; // cloud ice mass mixing ratio - uview_2d ni_dry; // dry cloud ice number mixing ratio - uview_2d w_updraft; // vertical wind velocity + uview_2d z_mid; // height at midpoints + uview_2d dz; // layer thickness + uview_2d qv_dry; // dry water vapor mixing ratio (dry air) + uview_2d qc_dry; // dry cloud water mass mixing ratio + uview_2d nc_dry; // dry cloud water number mixing ratio + uview_2d qi_dry; // cloud ice mass mixing ratio + uview_2d ni_dry; // dry cloud ice number mixing ratio + uview_2d w_updraft; // vertical wind velocity // aerosol dry interstitial/cloudborne number/mass mixing ratios // (because the number of species per mode varies, not all of these will @@ -348,125 +362,86 @@ struct Buffer { // number of local fields stored at column interfaces static constexpr int num_2d_iface = 1; - uview_2d z_iface; // height at interfaces + uview_2d z_iface; // height at interfaces // storage - Real* wsm_data; + Real *wsm_data; }; // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels inline size_t buffer_size(const int ncol, const int nlev) { - return sizeof(Real) * - (Buffer::num_2d_mid * ncol * nlev + - Buffer::num_2d_iface * ncol * (nlev+1)); + return sizeof(Real) * (Buffer::num_2d_mid * ncol * nlev + + Buffer::num_2d_iface * ncol * (nlev + 1)); } // ON HOST, initializeÑ• the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. inline size_t init_buffer(const ATMBufferManager &buffer_manager, - const int ncol, const int nlev, - Buffer &buffer) { - Real* mem = reinterpret_cast(buffer_manager.get_memory()); + const int ncol, const int nlev, Buffer &buffer) { + Real *mem = reinterpret_cast(buffer_manager.get_memory()); // set view pointers for midpoint fields - uview_2d* view_2d_mid_ptrs[Buffer::num_2d_mid] = { - &buffer.z_mid, - &buffer.dz, - &buffer.qv_dry, - &buffer.qc_dry, - &buffer.nc_dry, - &buffer.qi_dry, - &buffer.ni_dry, - &buffer.w_updraft, - - // aerosol modes - &buffer.dry_int_aero_nmr[0], - &buffer.dry_int_aero_nmr[1], - &buffer.dry_int_aero_nmr[2], - &buffer.dry_int_aero_nmr[3], - &buffer.dry_cld_aero_nmr[0], - &buffer.dry_cld_aero_nmr[1], - &buffer.dry_cld_aero_nmr[2], - &buffer.dry_cld_aero_nmr[3], - - // the following requires knowledge of mam4's mode-species layout - // (see mode_aero_species() in mam4xx/aero_modes.hpp) - - // accumulation mode - &buffer.dry_int_aero_mmr[0][0], - &buffer.dry_int_aero_mmr[0][1], - &buffer.dry_int_aero_mmr[0][2], - &buffer.dry_int_aero_mmr[0][3], - &buffer.dry_int_aero_mmr[0][4], - &buffer.dry_int_aero_mmr[0][5], - &buffer.dry_int_aero_mmr[0][6], - &buffer.dry_cld_aero_mmr[0][0], - &buffer.dry_cld_aero_mmr[0][1], - &buffer.dry_cld_aero_mmr[0][2], - &buffer.dry_cld_aero_mmr[0][3], - &buffer.dry_cld_aero_mmr[0][4], - &buffer.dry_cld_aero_mmr[0][5], - &buffer.dry_cld_aero_mmr[0][6], - - // aitken mode - &buffer.dry_int_aero_mmr[1][0], - &buffer.dry_int_aero_mmr[1][1], - &buffer.dry_int_aero_mmr[1][2], - &buffer.dry_int_aero_mmr[1][3], - &buffer.dry_cld_aero_mmr[1][0], - &buffer.dry_cld_aero_mmr[1][1], - &buffer.dry_cld_aero_mmr[1][2], - &buffer.dry_cld_aero_mmr[1][3], - - // coarse mode - &buffer.dry_int_aero_mmr[2][0], - &buffer.dry_int_aero_mmr[2][1], - &buffer.dry_int_aero_mmr[2][2], - &buffer.dry_int_aero_mmr[2][3], - &buffer.dry_int_aero_mmr[2][4], - &buffer.dry_int_aero_mmr[2][5], - &buffer.dry_int_aero_mmr[2][6], - &buffer.dry_cld_aero_mmr[2][0], - &buffer.dry_cld_aero_mmr[2][1], - &buffer.dry_cld_aero_mmr[2][2], - &buffer.dry_cld_aero_mmr[2][3], - &buffer.dry_cld_aero_mmr[2][4], - &buffer.dry_cld_aero_mmr[2][5], - &buffer.dry_cld_aero_mmr[2][6], - - // primary carbon mode - &buffer.dry_int_aero_mmr[3][0], - &buffer.dry_int_aero_mmr[3][1], - &buffer.dry_int_aero_mmr[3][2], - &buffer.dry_cld_aero_mmr[3][0], - &buffer.dry_cld_aero_mmr[3][1], - &buffer.dry_cld_aero_mmr[3][2], - - // aerosol gases - &buffer.dry_gas_mmr[0], - &buffer.dry_gas_mmr[1], - &buffer.dry_gas_mmr[2], - &buffer.dry_gas_mmr[3], - &buffer.dry_gas_mmr[4], - &buffer.dry_gas_mmr[5] - }; - for (int i = 0; i < Buffer::num_2d_scratch; ++i) { - view_2d_mid_ptrs[Buffer::num_2d_mid+i-Buffer::num_2d_scratch] = &buffer.scratch[i]; + uview_2d *view_2d_mid_ptrs[Buffer::num_2d_mid] = { + &buffer.z_mid, &buffer.dz, &buffer.qv_dry, &buffer.qc_dry, &buffer.nc_dry, + &buffer.qi_dry, &buffer.ni_dry, &buffer.w_updraft, + + // aerosol modes + &buffer.dry_int_aero_nmr[0], &buffer.dry_int_aero_nmr[1], + &buffer.dry_int_aero_nmr[2], &buffer.dry_int_aero_nmr[3], + &buffer.dry_cld_aero_nmr[0], &buffer.dry_cld_aero_nmr[1], + &buffer.dry_cld_aero_nmr[2], &buffer.dry_cld_aero_nmr[3], + + // the following requires knowledge of mam4's mode-species layout + // (see mode_aero_species() in mam4xx/aero_modes.hpp) + + // accumulation mode + &buffer.dry_int_aero_mmr[0][0], &buffer.dry_int_aero_mmr[0][1], + &buffer.dry_int_aero_mmr[0][2], &buffer.dry_int_aero_mmr[0][3], + &buffer.dry_int_aero_mmr[0][4], &buffer.dry_int_aero_mmr[0][5], + &buffer.dry_int_aero_mmr[0][6], &buffer.dry_cld_aero_mmr[0][0], + &buffer.dry_cld_aero_mmr[0][1], &buffer.dry_cld_aero_mmr[0][2], + &buffer.dry_cld_aero_mmr[0][3], &buffer.dry_cld_aero_mmr[0][4], + &buffer.dry_cld_aero_mmr[0][5], &buffer.dry_cld_aero_mmr[0][6], + + // aitken mode + &buffer.dry_int_aero_mmr[1][0], &buffer.dry_int_aero_mmr[1][1], + &buffer.dry_int_aero_mmr[1][2], &buffer.dry_int_aero_mmr[1][3], + &buffer.dry_cld_aero_mmr[1][0], &buffer.dry_cld_aero_mmr[1][1], + &buffer.dry_cld_aero_mmr[1][2], &buffer.dry_cld_aero_mmr[1][3], + + // coarse mode + &buffer.dry_int_aero_mmr[2][0], &buffer.dry_int_aero_mmr[2][1], + &buffer.dry_int_aero_mmr[2][2], &buffer.dry_int_aero_mmr[2][3], + &buffer.dry_int_aero_mmr[2][4], &buffer.dry_int_aero_mmr[2][5], + &buffer.dry_int_aero_mmr[2][6], &buffer.dry_cld_aero_mmr[2][0], + &buffer.dry_cld_aero_mmr[2][1], &buffer.dry_cld_aero_mmr[2][2], + &buffer.dry_cld_aero_mmr[2][3], &buffer.dry_cld_aero_mmr[2][4], + &buffer.dry_cld_aero_mmr[2][5], &buffer.dry_cld_aero_mmr[2][6], + + // primary carbon mode + &buffer.dry_int_aero_mmr[3][0], &buffer.dry_int_aero_mmr[3][1], + &buffer.dry_int_aero_mmr[3][2], &buffer.dry_cld_aero_mmr[3][0], + &buffer.dry_cld_aero_mmr[3][1], &buffer.dry_cld_aero_mmr[3][2], + + // aerosol gases + &buffer.dry_gas_mmr[0], &buffer.dry_gas_mmr[1], &buffer.dry_gas_mmr[2], + &buffer.dry_gas_mmr[3], &buffer.dry_gas_mmr[4], &buffer.dry_gas_mmr[5]}; + for(int i = 0; i < Buffer::num_2d_scratch; ++i) { + view_2d_mid_ptrs[Buffer::num_2d_mid + i - Buffer::num_2d_scratch] = + &buffer.scratch[i]; } - for (int i = 0; i < Buffer::num_2d_mid; ++i) { + for(int i = 0; i < Buffer::num_2d_mid; ++i) { *view_2d_mid_ptrs[i] = view_2d(mem, ncol, nlev); mem += view_2d_mid_ptrs[i]->size(); } // set view pointers for interface fields - uview_2d* view_2d_iface_ptrs[Buffer::num_2d_iface] = { - &buffer.z_iface - }; - for (int i = 0; i < Buffer::num_2d_iface; ++i) { - *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev+1); + uview_2d *view_2d_iface_ptrs[Buffer::num_2d_iface] = {&buffer.z_iface}; + for(int i = 0; i < Buffer::num_2d_iface; ++i) { + *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev + 1); mem += view_2d_iface_ptrs[i]->size(); } @@ -476,47 +451,48 @@ inline size_t init_buffer(const ATMBufferManager &buffer_manager, /* // Compute workspace manager size to check used memory vs. requested memory // (if needed) - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); const int n_wind_slots = ekat::npack(2)*Spack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); - mem += wsm_size; + const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, + 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); mem += wsm_size; */ // return the number of bytes allocated - return (mem - buffer_manager.get_memory())*sizeof(Real); + return (mem - buffer_manager.get_memory()) * sizeof(Real); } // Given a dry atmosphere state, creates a haero::Atmosphere object for the // column with the given index. This object can be provided to mam4xx for the // column. KOKKOS_INLINE_FUNCTION -haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, +haero::Atmosphere atmosphere_for_column(const DryAtmosphere &dry_atm, const int column_index) { EKAT_KERNEL_ASSERT_MSG(dry_atm.T_mid.data() != nullptr, - "T_mid not defined for dry atmosphere state!"); + "T_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_mid.data() != nullptr, - "p_mid not defined for dry atmosphere state!"); + "p_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qv.data() != nullptr, - "qv not defined for dry atmosphere state!"); + "qv not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qc.data() != nullptr, - "qc not defined for dry atmosphere state!"); + "qc not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.nc.data() != nullptr, - "nc not defined for dry atmosphere state!"); + "nc not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qi.data() != nullptr, - "qi not defined for dry atmosphere state!"); + "qi not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.ni.data() != nullptr, - "ni not defined for dry atmosphere state!"); + "ni not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.z_mid.data() != nullptr, - "z_mid not defined for dry atmosphere state!"); + "z_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del.data() != nullptr, - "p_del not defined for dry atmosphere state!"); + "p_del not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_int.data() != nullptr, - "p_int not defined for dry atmosphere state!"); + "p_int not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.cldfrac.data() != nullptr, - "cldfrac not defined for dry atmosphere state!"); + "cldfrac not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, - "w_updraft not defined for dry atmosphere state!"); + "w_updraft not defined for dry atmosphere state!"); return haero::Atmosphere(mam4::nlev, ekat::subview(dry_atm.T_mid, column_index), ekat::subview(dry_atm.p_mid, column_index), @@ -538,23 +514,24 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, // ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to // mam4xx for the column. KOKKOS_INLINE_FUNCTION -mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, +mam4::Prognostics interstitial_aerosols_for_column(const AerosolState &dry_aero, const int column_index) { constexpr int nlev = mam4::nlev; mam4::Prognostics progs(nlev); - for (int m = 0; m < num_aero_modes(); ++m) { + for(int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), - "int_aero_nmr not defined for dry aerosol state!"); + "int_aero_nmr not defined for dry aerosol state!"); progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); - for (int a = 0; a < num_aero_species(); ++a) { - if (dry_aero.int_aero_mmr[m][a].data()) { - progs.q_aero_i[m][a] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + for(int a = 0; a < num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + progs.q_aero_i[m][a] = + ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); } } } - for (int g = 0; g < num_aero_gases(); ++g) { + for(int g = 0; g < num_aero_gases(); ++g) { EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), - "gas_mmr not defined for dry aerosol state!"); + "gas_mmr not defined for dry aerosol state!"); progs.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); } return progs; @@ -564,16 +541,17 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, // with the given index with interstitial and cloudborne aerosol views defined. // This object can be provided to mam4xx for the column. KOKKOS_INLINE_FUNCTION -mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, +mam4::Prognostics aerosols_for_column(const AerosolState &dry_aero, const int column_index) { auto progs = interstitial_aerosols_for_column(dry_aero, column_index); - for (int m = 0; m < num_aero_modes(); ++m) { + for(int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), - "dry_cld_aero_nmr not defined for aerosol state!"); + "dry_cld_aero_nmr not defined for aerosol state!"); progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); - for (int a = 0; a < num_aero_species(); ++a) { - if (dry_aero.cld_aero_mmr[m][a].data()) { - progs.q_aero_c[m][a] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + for(int a = 0; a < num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + progs.q_aero_c[m][a] = + ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); } } } @@ -599,26 +577,27 @@ void compute_vertical_layer_heights(const Team& team, }*/ KOKKOS_INLINE_FUNCTION -void compute_vertical_layer_heights(const Team& team, - const DryAtmosphere& dry_atm, +void compute_vertical_layer_heights(const Team &team, + const DryAtmosphere &dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); - const auto dz = ekat::subview(dry_atm.dz, column_index); - const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine + const auto dz = ekat::subview(dry_atm.dz, column_index); + const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); - const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); - const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); - const auto qv = ekat::subview(dry_atm.qv, column_index); + const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); + const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); - PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs - dz);//output + PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs + dz); // output team.team_barrier(); - PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs - z_iface); //output - team.team_barrier(); // likely necessary to have z_iface up to date + PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, // inputs + z_iface); // output + team.team_barrier(); // likely necessary to have z_iface up to date PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } @@ -626,19 +605,23 @@ void compute_vertical_layer_heights(const Team& team, // team to compute the vertical updraft velocity for the column with the given // index. KOKKOS_INLINE_FUNCTION -void compute_updraft_velocities(const Team& team, - const WetAtmosphere& wet_atm, - const DryAtmosphere& dry_atm, +void compute_updraft_velocities(const Team &team, const WetAtmosphere &wet_atm, + const DryAtmosphere &dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - dry_atm.dz(i,k) = PF::calculate_dz(dry_atm.p_del(i,k), dry_atm.p_mid(i,k), dry_atm.T_mid(i,k), wet_atm.qv(i,k)); - const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); - dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { + dry_atm.dz(i, k) = + PF::calculate_dz(dry_atm.p_del(i, k), dry_atm.p_mid(i, k), + dry_atm.T_mid(i, k), wet_atm.qv(i, k)); + const auto rho = + PF::calculate_density(dry_atm.p_del(i, k), dry_atm.dz(i, k)); + dry_atm.w_updraft(i, k) = + PF::calculate_vertical_velocity(wet_atm.omega(i, k), rho); }); } @@ -646,93 +629,108 @@ void compute_updraft_velocities(const Team& team, // from the team to compute mixing ratios for a dry atmosphere state in th // column with the given index. KOKKOS_INLINE_FUNCTION -void compute_dry_mixing_ratios(const Team& team, - const WetAtmosphere& wet_atm, - const DryAtmosphere& dry_atm, +void compute_dry_mixing_ratios(const Team &team, const WetAtmosphere &wet_atm, + const DryAtmosphere &dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { + const auto qv_ik = wet_atm.qv(i, k); + dry_atm.qv(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i, k), qv_ik); + dry_atm.qc(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i, k), qv_ik); + dry_atm.nc(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i, k), qv_ik); + dry_atm.qi(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i, k), qv_ik); + dry_atm.ni(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i, k), qv_ik); }); } -// Given a thread team and wet atmospheric and aerosol states, dispatches threads -// from the team to compute mixing ratios for the given dry interstitial aerosol -// state for the column with the given index. +// Given a thread team and wet atmospheric and aerosol states, dispatches +// threads from the team to compute mixing ratios for the given dry interstitial +// aerosol state for the column with the given index. KOKKOS_INLINE_FUNCTION -void compute_dry_mixing_ratios(const Team& team, - const WetAtmosphere& wet_atm, - const AerosolState& wet_aero, - const AerosolState& dry_aero, +void compute_dry_mixing_ratios(const Team &team, const WetAtmosphere &wet_atm, + const AerosolState &wet_aero, + const AerosolState &dry_aero, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - const auto qv_ik = wet_atm.qv(i,k); - for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); - if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { + const auto qv_ik = wet_atm.qv(i, k); + for(int m = 0; m < num_aero_modes(); ++m) { + dry_aero.int_aero_nmr[m](i, k) = PF::calculate_drymmr_from_wetmmr( + wet_aero.int_aero_nmr[m](i, k), qv_ik); + if(dry_aero.cld_aero_nmr[m].data()) { + dry_aero.cld_aero_nmr[m](i, k) = PF::calculate_drymmr_from_wetmmr( + wet_aero.cld_aero_nmr[m](i, k), qv_ik); } - for (int a = 0; a < num_aero_species(); ++a) { - if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + for(int a = 0; a < num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + dry_aero.int_aero_mmr[m][a](i, k) = PF::calculate_drymmr_from_wetmmr( + wet_aero.int_aero_mmr[m][a](i, k), qv_ik); } - if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + if(dry_aero.cld_aero_mmr[m][a].data()) { + dry_aero.cld_aero_mmr[m][a](i, k) = PF::calculate_drymmr_from_wetmmr( + wet_aero.cld_aero_mmr[m][a](i, k), qv_ik); } } } - for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + for(int g = 0; g < num_aero_gases(); ++g) { + dry_aero.gas_mmr[g](i, k) = + PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i, k), qv_ik); } }); } -// Given a thread team and dry atmospheric and aerosol states, dispatches threads -// from the team to compute mixing ratios for the given wet interstitial aerosol -// state for the column with the given index. +// Given a thread team and dry atmospheric and aerosol states, dispatches +// threads from the team to compute mixing ratios for the given wet interstitial +// aerosol state for the column with the given index. KOKKOS_INLINE_FUNCTION -void compute_wet_mixing_ratios(const Team& team, - const DryAtmosphere& dry_atm, - const AerosolState& dry_aero, - const AerosolState& wet_aero, +void compute_wet_mixing_ratios(const Team &team, const DryAtmosphere &dry_atm, + const AerosolState &dry_aero, + const AerosolState &wet_aero, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - const auto qv_ik = dry_atm.qv(i,k); - for (int m = 0; m < num_aero_modes(); ++m) { - wet_aero.int_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_nmr[m](i,k), qv_ik); - if (wet_aero.cld_aero_nmr[m].data()) { - wet_aero.cld_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_nmr[m](i,k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { + const auto qv_ik = dry_atm.qv(i, k); + for(int m = 0; m < num_aero_modes(); ++m) { + wet_aero.int_aero_nmr[m](i, k) = PF::calculate_wetmmr_from_drymmr( + dry_aero.int_aero_nmr[m](i, k), qv_ik); + if(wet_aero.cld_aero_nmr[m].data()) { + wet_aero.cld_aero_nmr[m](i, k) = PF::calculate_wetmmr_from_drymmr( + dry_aero.cld_aero_nmr[m](i, k), qv_ik); } - for (int a = 0; a < num_aero_species(); ++a) { - if (wet_aero.int_aero_mmr[m][a].data()) { - wet_aero.int_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_mmr[m][a](i,k), qv_ik); + for(int a = 0; a < num_aero_species(); ++a) { + if(wet_aero.int_aero_mmr[m][a].data()) { + wet_aero.int_aero_mmr[m][a](i, k) = PF::calculate_wetmmr_from_drymmr( + dry_aero.int_aero_mmr[m][a](i, k), qv_ik); } - if (wet_aero.cld_aero_mmr[m][a].data()) { - wet_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_mmr[m][a](i,k), qv_ik); + if(wet_aero.cld_aero_mmr[m][a].data()) { + wet_aero.cld_aero_mmr[m][a](i, k) = PF::calculate_wetmmr_from_drymmr( + dry_aero.cld_aero_mmr[m][a](i, k), qv_ik); } } } - for (int g = 0; g < num_aero_gases(); ++g) { - wet_aero.gas_mmr[g](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i,k), qv_ik); + for(int g = 0; g < num_aero_gases(); ++g) { + wet_aero.gas_mmr[g](i, k) = + PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i, k), qv_ik); } }); } @@ -742,50 +740,55 @@ void compute_wet_mixing_ratios(const Team& team, // these constants where needed within two such functions so we don't define // them inconsistently. Yes, it's the 21st century and we're still struggling // with these basic things. -#define DECLARE_PROG_TRANSFER_CONSTANTS \ - /* mapping of constituent indices to aerosol modes */ \ - const auto Accum = mam4::ModeIndex::Accumulation; \ - const auto Aitken = mam4::ModeIndex::Aitken; \ - const auto Coarse = mam4::ModeIndex::Coarse; \ - const auto PC = mam4::ModeIndex::PrimaryCarbon; \ - const auto NoMode = mam4::ModeIndex::None; \ - static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ - Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ - Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ - Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ - PC, PC, PC, PC, /* 3 aero species + NMR */ \ - }; \ - /* mapping of constituent indices to aerosol species */ \ - const auto SOA = mam4::AeroId::SOA; \ - const auto SO4 = mam4::AeroId::SO4; \ - const auto POM = mam4::AeroId::POM; \ - const auto BC = mam4::AeroId::BC; \ - const auto NaCl = mam4::AeroId::NaCl; \ - const auto DST = mam4::AeroId::DST; \ - const auto MOM = mam4::AeroId::MOM; \ - const auto NoAero = mam4::AeroId::None; \ - static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ - NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) */ \ - SO4, POM, SOA, BC, DST, NaCl, MOM, NoAero, /* accumulation mode */ \ - SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ - DST, NaCl, SO4, BC, POM, SOA, MOM, NoAero, /* coarse mode */ \ - POM, BC, MOM, NoAero, /* primary carbon mode */ \ - }; \ - /* mapping of constituent indices to gases */ \ - const auto O3 = mam4::GasId::O3; \ - const auto H2O2 = mam4::GasId::H2O2; \ - const auto H2SO4 = mam4::GasId::H2SO4; \ - const auto SO2 = mam4::GasId::SO2; \ - const auto DMS = mam4::GasId::DMS; \ - const auto SOAG = mam4::GasId::SOAG; \ - const auto NoGas = mam4::GasId::None; \ - static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ - O3, H2O2, H2SO4, SO2, DMS, SOAG, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, \ +#define DECLARE_PROG_TRANSFER_CONSTANTS \ + /* mapping of constituent indices to aerosol modes */ \ + const auto Accum = mam4::ModeIndex::Accumulation; \ + const auto Aitken = mam4::ModeIndex::Aitken; \ + const auto Coarse = mam4::ModeIndex::Coarse; \ + const auto PC = mam4::ModeIndex::PrimaryCarbon; \ + const auto NoMode = mam4::ModeIndex::None; \ + static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) \ + */ \ + Accum, Accum, Accum, Accum, Accum, Accum, Accum, \ + Accum, /* 7 aero species + NMR */ \ + Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ + Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, \ + Coarse, /* 7 aero species + NMR */ \ + PC, PC, PC, PC, /* 3 aero species + NMR */ \ + }; \ + /* mapping of constituent indices to aerosol species */ \ + const auto SOA = mam4::AeroId::SOA; \ + const auto SO4 = mam4::AeroId::SO4; \ + const auto POM = mam4::AeroId::POM; \ + const auto BC = mam4::AeroId::BC; \ + const auto NaCl = mam4::AeroId::NaCl; \ + const auto DST = mam4::AeroId::DST; \ + const auto MOM = mam4::AeroId::MOM; \ + const auto NoAero = mam4::AeroId::None; \ + static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ + NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) \ + */ \ + SO4, POM, SOA, BC, DST, NaCl, MOM, \ + NoAero, /* accumulation mode */ \ + SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ + DST, NaCl, SO4, BC, POM, SOA, MOM, \ + NoAero, /* coarse mode */ \ + POM, BC, MOM, NoAero, /* primary carbon mode */ \ + }; \ + /* mapping of constituent indices to gases */ \ + const auto O3 = mam4::GasId::O3; \ + const auto H2O2 = mam4::GasId::H2O2; \ + const auto H2SO4 = mam4::GasId::H2SO4; \ + const auto SO2 = mam4::GasId::SO2; \ + const auto DMS = mam4::GasId::DMS; \ + const auto SOAG = mam4::GasId::SOAG; \ + const auto NoGas = mam4::GasId::None; \ + static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ + O3, H2O2, H2SO4, SO2, DMS, SOAG, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ }; // Given a Prognostics object, transfers data for interstitial aerosols to the @@ -798,30 +801,29 @@ void compute_wet_mixing_ratios(const Team& team, // NOTE: indices KOKKOS_INLINE_FUNCTION void transfer_prognostics_to_work_arrays(const mam4::Prognostics &progs, - const int k, - Real q[gas_pcnst()], + const int k, Real q[gas_pcnst()], Real qqcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS // copy number/mass mixing ratios from progs to q and qqcw at level k, // converting them to VMR - for (int i = 0; i < gas_pcnst(); ++i) { + for(int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); - q[i] = progs.q_gas[g](k); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if(gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + q[i] = progs.q_gas[g](k); qqcw[i] = progs.q_gas[g](k); } else { int m = static_cast(mode_index); - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); - q[i] = progs.q_aero_i[m][a](k); + if(aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + q[i] = progs.q_aero_i[m][a](k); qqcw[i] = progs.q_aero_c[m][a](k); - } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); - q[i] = progs.n_mode_i[m](k); + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); + q[i] = progs.n_mode_i[m](k); qqcw[i] = progs.n_mode_c[m](k); } } @@ -837,23 +839,29 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], Real vmrcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS - for (int i = 0; i < gas_pcnst(); ++i) { + for(int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if(gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); const Real mw = mam4::gas_species(g).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { +<<<<<<< HEAD if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); +======= + int m = static_cast(mode_index); + if(aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); +>>>>>>> Modified species order to match E3SM with the folling change in mam4xx const Real mw = mam4::aero_species(a).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); - } else { // constituent is a modal number mixing ratio - vmr[i] = q[i]; + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + } else { // constituent is a modal number mixing ratio + vmr[i] = q[i]; vmrcw[i] = qqcw[i]; } } @@ -865,27 +873,32 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], KOKKOS_INLINE_FUNCTION void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], const Real vmrcw[gas_pcnst()], - Real q[gas_pcnst()], - Real qqcw[gas_pcnst()]) { + Real q[gas_pcnst()], Real qqcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS - for (int i = 0; i < gas_pcnst(); ++i) { + for(int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if(gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); const Real mw = mam4::gas_species(g).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { +<<<<<<< HEAD if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); +======= + int m = static_cast(mode_index); + if(aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); +>>>>>>> Modified species order to match E3SM with the folling change in mam4xx const Real mw = mam4::aero_species(a).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); - } else { // constituent is a modal number mixing ratio - q[i] = vmr[i]; + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + } else { // constituent is a modal number mixing ratio + q[i] = vmr[i]; qqcw[i] = vmrcw[i]; } } @@ -898,25 +911,31 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], KOKKOS_INLINE_FUNCTION void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], const Real qqcw[gas_pcnst()], - mam4::Prognostics &progs, const int k) { + mam4::Prognostics &progs, + const int k) { DECLARE_PROG_TRANSFER_CONSTANTS // copy number/mass mixing ratios from progs to q and qqcw at level k, // converting them to VMR - for (int i = 0; i < gas_pcnst(); ++i) { + for(int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if(gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); progs.q_gas[g](k) = q[i]; } else { int m = static_cast(mode_index); - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); + if(aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); progs.q_aero_i[m][a](k) = q[i]; progs.q_aero_c[m][a](k) = qqcw[i]; +<<<<<<< HEAD } else { // constituent is a modal number mixing ratio +======= + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); +>>>>>>> Modified species order to match E3SM with the folling change in mam4xx progs.n_mode_i[m](k) = q[i]; progs.n_mode_c[m](k) = qqcw[i]; } @@ -926,6 +945,6 @@ void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], #undef DECLARE_PROG_TRANSFER_CONSTANTS -} // namespace scream::mam_coupling +} // namespace scream::mam_coupling #endif diff --git a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml index 946d11dcc0e..298da6d50ad 100644 --- a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml @@ -6,7 +6,8 @@ driver_options: time_stepping: time_step: ${ATM_TIME_STEP} run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} + #number_of_steps: ${NUM_STEPS} + number_of_steps: 1 atmosphere_processes: atm_procs_list: [mam4_aci] @@ -22,8 +23,8 @@ grids_manager: number_of_vertical_levels: 72 initial_conditions: - # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + # The name of the file containing the initial conditions for this test. FIXME: use a variable for the filename here + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc T_mid: 273.0 p_mid: 1.e5 p_int: 1.e5 From b841ffc427b7adbaa1bb49782f666d69a857e9a0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 23 Mar 2024 12:49:58 -0700 Subject: [PATCH 041/142] Reverted back mam_coupling changes for indexing but added a func to get prog index Changes for formatting and species reordering are reverted back in mam_coupling. Adds a new function in mam_couplingto map indexing to the prognostic array index. I changed the following mam4xx/aero_modes.cpp function (this is the only change in this file) to change species ordering: KOKKOS_INLINE_FUNCTION AeroId mode_aero_species(const int modeNo, const int speciesNo) { // A list of species within each mode for MAM4. static constexpr AeroId mode_aero_species[4][7] = { {// accumulation mode AeroId::SO4, AeroId::POM, AeroId::SOA, AeroId::BC, AeroId::DST, AeroId::NaCl, AeroId::MOM}, { // aitken mode AeroId::SO4, AeroId::SOA, AeroId::NaCl, AeroId::MOM, AeroId::None, AeroId::None, AeroId::None, }, {// coarse mode AeroId::DST, AeroId::NaCl, AeroId::SO4, AeroId::BC, AeroId::POM, AeroId::SOA, AeroId::MOM}, {// primary carbon mode AeroId::POM, AeroId::BC, AeroId::MOM, AeroId::None, AeroId::None, AeroId::None, AeroId::None}}; return mode_aero_species[modeNo][speciesNo]; } --- .../mam/eamxx_mam_aci_process_interface.cpp | 15 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 857 ++++++++++-------- 2 files changed, 466 insertions(+), 406 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ee8fb3b130e..2e32e067077 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -686,11 +686,11 @@ void call_hetfrz_compute_tendencies( diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + // diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); // naai and naai_hom are the outputs needed for nucleate_ice and these // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + /*diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); //------------------------------------------------------------- @@ -703,10 +703,11 @@ void call_hetfrz_compute_tendencies( // values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; - const Real t = 0, dt = 0; - hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, progs, - diags, tends); - //}); + const Real t = 0, dt = 0;*/ + // hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, + // progs, + // diags, tends); + // }); } } } // namespace @@ -920,6 +921,8 @@ void MAMAci::set_grids( const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + // std::cout << "--> Gasses:" << g <<" spec:"< -#include #include +#include +#include #include #include @@ -13,7 +13,7 @@ namespace scream::mam_coupling { -using KT = ekat::KokkosTypes; +using KT = ekat::KokkosTypes; // views for single- and multi-column data using view_1d = typename KT::template view_1d; @@ -27,10 +27,8 @@ using const_view_3d = typename KT::template view_3d; using Team = Kokkos::TeamPolicy::member_type; // unmanaged views (for buffer and workspace manager) -using uview_1d = - typename ekat::template Unmanaged>; -using uview_2d = - typename ekat::template Unmanaged>; +using uview_1d = typename ekat::template Unmanaged>; +using uview_2d = typename ekat::template Unmanaged>; using PF = scream::PhysicsFunctions; @@ -47,15 +45,21 @@ constexpr int nqtendbb() { return 4; } // returns the number of distinct aerosol modes KOKKOS_INLINE_FUNCTION -constexpr int num_aero_modes() { return mam4::AeroConfig::num_modes(); } +constexpr int num_aero_modes() { + return mam4::AeroConfig::num_modes(); +} // returns the number of distinct aerosol species KOKKOS_INLINE_FUNCTION -constexpr int num_aero_species() { return mam4::AeroConfig::num_aerosol_ids(); } +constexpr int num_aero_species() { + return mam4::AeroConfig::num_aerosol_ids(); +} // returns the number of distinct aerosol-related gases KOKKOS_INLINE_FUNCTION -constexpr int num_aero_gases() { return mam4::AeroConfig::num_gas_ids(); } +constexpr int num_aero_gases() { + return mam4::AeroConfig::num_gas_ids(); +} // returns the total number of aerosol tracers (i.e. the total number of // distinct valid mode-species pairs) @@ -69,12 +73,12 @@ constexpr int num_aero_tracers() { // Given a MAM aerosol mode index, returns a string denoting the symbolic // name of the mode. KOKKOS_INLINE_FUNCTION -const char *aero_mode_name(const int mode) { +const char* aero_mode_name(const int mode) { static const char *mode_names[num_aero_modes()] = { - "1", - "2", - "3", - "4", + "1", + "2", + "3", + "4", }; return mode_names[mode]; } @@ -82,18 +86,31 @@ const char *aero_mode_name(const int mode) { // Given a MAM aerosol species ID, returns a string denoting the symbolic // name of the species. KOKKOS_INLINE_FUNCTION -const char *aero_species_name(const int species_id) { +const char* aero_species_name(const int species_id) { static const char *species_names[num_aero_species()] = { - "so4", "pom", "soa", "bc", "dst", "nacl", "mom"}; + "soa", + "so4", + "pom", + "bc", + "nacl", + "dst", + "mom", + }; return species_names[species_id]; } // Given a MAM aerosol-related gas ID, returns a string denoting the symbolic // name of the gas species. KOKKOS_INLINE_FUNCTION -const char *gas_species_name(const int gas_id) { - static const char *species_names[num_aero_gases()] = {"o3", "h2o2", "h2so4", - "so2", "dms", "soag"}; +const char* gas_species_name(const int gas_id) { + static const char *species_names[num_aero_gases()] = { + "o3", + "h2o2", + "h2so4", + "so2", + "dms", + "soag" + }; return species_names[gas_id]; } @@ -101,155 +118,158 @@ const char *gas_species_name(const int gas_id) { namespace { KOKKOS_INLINE_FUNCTION -constexpr int max_field_name_len() { return 128; } +constexpr int max_field_name_len() { + return 128; +} KOKKOS_INLINE_FUNCTION -size_t gpu_strlen(const char *s) { +size_t gpu_strlen(const char* s) { size_t l = 0; - while(s[l]) ++l; + while (s[l]) ++l; return l; } KOKKOS_INLINE_FUNCTION void concat_2_strings(const char *s1, const char *s2, char *concatted) { size_t len1 = gpu_strlen(s1); - for(size_t i = 0; i < len1; ++i) concatted[i] = s1[i]; + for (size_t i = 0; i < len1; ++i) + concatted[i] = s1[i]; size_t len2 = gpu_strlen(s2); - for(size_t i = 0; i < len2; ++i) concatted[i + len1] = s2[i]; - concatted[len1 + len2] = 0; + for (size_t i = 0; i < len2; ++i) + concatted[i + len1] = s2[i]; + concatted[len1+len2] = 0; } KOKKOS_INLINE_FUNCTION -void concat_3_strings(const char *s1, const char *s2, const char *s3, - char *concatted) { +void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { size_t len1 = gpu_strlen(s1); - for(size_t i = 0; i < len1; ++i) concatted[i] = s1[i]; + for (size_t i = 0; i < len1; ++i) + concatted[i] = s1[i]; size_t len2 = gpu_strlen(s2); - for(size_t i = 0; i < len2; ++i) concatted[i + len1] = s2[i]; + for (size_t i = 0; i < len2; ++i) + concatted[i + len1] = s2[i]; size_t len3 = gpu_strlen(s3); - for(size_t i = 0; i < len3; ++i) concatted[i + len1 + len2] = s3[i]; - concatted[len1 + len2 + len3] = 0; + for (size_t i = 0; i < len3; ++i) + concatted[i + len1 + len2] = s3[i]; + concatted[len1+len2+len3] = 0; } KOKKOS_INLINE_FUNCTION -char *int_aero_nmr_names(int mode) { +char* int_aero_nmr_names(int mode) { static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return int_aero_nmr_names_[mode]; } KOKKOS_INLINE_FUNCTION -char *cld_aero_nmr_names(int mode) { +char* cld_aero_nmr_names(int mode) { static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return cld_aero_nmr_names_[mode]; } KOKKOS_INLINE_FUNCTION -char *int_aero_mmr_names(int mode, int species) { - static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()] - [max_field_name_len()] = {}; +char* int_aero_mmr_names(int mode, int species) { + static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return int_aero_mmr_names_[mode][species]; } KOKKOS_INLINE_FUNCTION -char *cld_aero_mmr_names(int mode, int species) { - static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()] - [max_field_name_len()] = {}; +char* cld_aero_mmr_names(int mode, int species) { + static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return cld_aero_mmr_names_[mode][species]; } KOKKOS_INLINE_FUNCTION -char *gas_mmr_names(int gas_id) { +char* gas_mmr_names(int gas_id) { static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; return gas_mmr_names_[gas_id]; } -} // end anonymous namespace +} // end anonymous namespace // Given a MAM aerosol mode index, returns the name of the related interstitial // modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") KOKKOS_INLINE_FUNCTION -const char *int_aero_nmr_field_name(const int mode) { - if(!int_aero_nmr_names(mode)[0]) { +const char* int_aero_nmr_field_name(const int mode) { + if (!int_aero_nmr_names(mode)[0]) { concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); } - return const_cast(int_aero_nmr_names(mode)); + return const_cast(int_aero_nmr_names(mode)); } // Given a MAM aerosol mode index, returns the name of the related cloudborne // modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") KOKKOS_INLINE_FUNCTION -const char *cld_aero_nmr_field_name(const int mode) { - if(!cld_aero_nmr_names(mode)[0]) { +const char* cld_aero_nmr_field_name(const int mode) { + if (!cld_aero_nmr_names(mode)[0]) { concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); } - return const_cast(cld_aero_nmr_names(mode)); + return const_cast(cld_aero_nmr_names(mode)); } // Given a MAM aerosol mode index and the index of the MAM aerosol species // within it, returns the name of the relevant interstitial mass mixing ratio -// field in EAMxx. The form of the field name is -// "_a<1-based-mode-index>". If the desired species is not present -// within the desire mode, returns a blank string (""). +// field in EAMxx. The form of the field name is "_a<1-based-mode-index>". +// If the desired species is not present within the desire mode, returns a blank +// string (""). KOKKOS_INLINE_FUNCTION -const char *int_aero_mmr_field_name(const int mode, const int species) { - if(!int_aero_mmr_names(mode, species)[0]) { +const char* int_aero_mmr_field_name(const int mode, const int species) { + if (!int_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); - if(aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), "_a", - aero_mode_name(mode), int_aero_mmr_names(mode, species)); + if (aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), + "_a", aero_mode_name(mode), + int_aero_mmr_names(mode, species)); } } - return const_cast(int_aero_mmr_names(mode, species)); + return const_cast(int_aero_mmr_names(mode, species)); }; // Given a MAM aerosol mode index and the index of the MAM aerosol species // within it, returns the name of the relevant cloudborne mass mixing ratio -// field in EAMxx. The form of the field name is -// "_c<1-based-mode-index>". If the desired species is not present -// within the desire mode, returns a blank string (""). +// field in EAMxx. The form of the field name is "_c<1-based-mode-index>". +// If the desired species is not present within the desire mode, returns a blank +// string (""). KOKKOS_INLINE_FUNCTION -const char *cld_aero_mmr_field_name(const int mode, const int species) { - if(!cld_aero_mmr_names(mode, species)[0]) { +const char* cld_aero_mmr_field_name(const int mode, const int species) { + if (!cld_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); - if(aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), "_c", - aero_mode_name(mode), cld_aero_mmr_names(mode, species)); + if (aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), + "_c", aero_mode_name(mode), + cld_aero_mmr_names(mode, species)); } } - return const_cast(cld_aero_mmr_names(mode, species)); + return const_cast(cld_aero_mmr_names(mode, species)); }; // Given a MAM aerosol-related gas identifier, returns the name of its mass // mixing ratio field in EAMxx ("aero_gas_mmr_") KOKKOS_INLINE_FUNCTION -const char *gas_mmr_field_name(const int gas) { - if(!gas_mmr_names(gas)[0]) { - concat_2_strings("aero_gas_mmr_", gas_species_name(gas), - gas_mmr_names(gas)); +const char* gas_mmr_field_name(const int gas) { + if (!gas_mmr_names(gas)[0]) { + concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); } - return const_cast(gas_mmr_names(gas)); + return const_cast(gas_mmr_names(gas)); } // This type stores multi-column views related specifically to the wet // atmospheric state used by EAMxx. struct WetAtmosphere { - const_view_2d - qv; // wet water vapor specific humidity [kg vapor / kg moist air] - const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud - // water/kg moist air] - const_view_2d - nc; // wet cloud liquid water number mixing ratio [# / kg moist air] - const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice - // water / kg moist air] - const_view_2d - ni; // wet cloud ice water number mixing ratio [# / kg moist air] - const_view_2d omega; // vertical pressure velocity [Pa/s] + const_view_2d qv; // wet water vapor specific humidity [kg vapor / kg moist air] + const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud water/kg moist air] + const_view_2d nc; // wet cloud liquid water number mixing ratio [# / kg moist air] + const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice water / kg moist air] + const_view_2d ni; // wet cloud ice water number mixing ratio [# / kg moist air] + const_view_2d omega; // vertical pressure velocity [Pa/s] }; // This type stores multi-column views related to the dry atmospheric state // used by MAM. struct DryAtmosphere { <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index Real z_surf; // height of bottom of atmosphere [m] const_view_2d T_mid; // temperature at grid midpoints [K] const_view_2d p_mid; // total pressure at grid midpoints [Pa] @@ -262,11 +282,16 @@ struct DryAtmosphere { view_2d z_iface; // height at layer interfaces [m] view_2d dz; // layer thickness [m] const_view_2d p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] +<<<<<<< HEAD const_view_2d p_int; // total pressure at grid interfaces [Pa] +======= + const_view_2d p_int; // total pressure at grid interfaces [Pa] +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const_view_2d cldfrac; // cloud fraction [-] view_2d w_updraft; // updraft velocity [m/s] const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] +<<<<<<< HEAD ======= Real z_surf; // height of bottom of atmosphere [m] const_view_2d T_mid; // temperature at grid midpoints [K] @@ -289,28 +314,22 @@ struct DryAtmosphere { const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] >>>>>>> Modified species order to match E3SM with the folling change in mam4xx +======= +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index }; // This type stores aerosol number and mass mixing ratios evolved by MAM. It // can be used to represent wet and dry aerosols. When you declare an // AerosolState, you must decide whether it's a dry or wet aerosol state (with // mixing ratios in terms of dry or wet parcels of air, respectively). -// These mixing ratios are organized by mode (and species, for mass mixing -// ratio) in the same way as they are in mam4xx, and indexed using -// mam4::AeroConfig. +// These mixing ratios are organized by mode (and species, for mass mixing ratio) +// in the same way as they are in mam4xx, and indexed using mam4::AeroConfig. struct AerosolState { - view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number - // mixing ratios [# / kg air] - view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number - // mixing ratios [# / kg air] - view_2d int_aero_mmr[num_aero_modes()] - [num_aero_species()]; // interstitial aerosol mass mixing - // ratios [kg aerosol / kg air] - view_2d cld_aero_mmr[num_aero_modes()] - [num_aero_species()]; // cloudborne aerosol mass mixing - // ratios [kg aerosol / kg air] - view_2d - gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] + view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number mixing ratios [# / kg air] + view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number mixing ratios [# / kg air] + view_2d int_aero_mmr[num_aero_modes()][num_aero_species()]; // interstitial aerosol mass mixing ratios [kg aerosol / kg air] + view_2d cld_aero_mmr[num_aero_modes()][num_aero_species()]; // cloudborne aerosol mass mixing ratios [kg aerosol / kg air] + view_2d gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] }; // storage for variables used within MAM atmosphere processes, initialized with @@ -326,20 +345,20 @@ struct Buffer { static constexpr int num_2d_scratch = 10; // number of local fields stored at column midpoints - static constexpr int num_2d_mid = - 8 + // number of dry atm fields - 2 * (num_aero_modes() + num_aero_tracers()) + num_aero_gases() + - num_2d_scratch; + static constexpr int num_2d_mid = 8 + // number of dry atm fields + 2 * (num_aero_modes() + num_aero_tracers()) + + num_aero_gases() + + num_2d_scratch; // (dry) atmospheric state - uview_2d z_mid; // height at midpoints - uview_2d dz; // layer thickness - uview_2d qv_dry; // dry water vapor mixing ratio (dry air) - uview_2d qc_dry; // dry cloud water mass mixing ratio - uview_2d nc_dry; // dry cloud water number mixing ratio - uview_2d qi_dry; // cloud ice mass mixing ratio - uview_2d ni_dry; // dry cloud ice number mixing ratio - uview_2d w_updraft; // vertical wind velocity + uview_2d z_mid; // height at midpoints + uview_2d dz; // layer thickness + uview_2d qv_dry; // dry water vapor mixing ratio (dry air) + uview_2d qc_dry; // dry cloud water mass mixing ratio + uview_2d nc_dry; // dry cloud water number mixing ratio + uview_2d qi_dry; // cloud ice mass mixing ratio + uview_2d ni_dry; // dry cloud ice number mixing ratio + uview_2d w_updraft; // vertical wind velocity // aerosol dry interstitial/cloudborne number/mass mixing ratios // (because the number of species per mode varies, not all of these will @@ -362,86 +381,125 @@ struct Buffer { // number of local fields stored at column interfaces static constexpr int num_2d_iface = 1; - uview_2d z_iface; // height at interfaces + uview_2d z_iface; // height at interfaces // storage - Real *wsm_data; + Real* wsm_data; }; // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels inline size_t buffer_size(const int ncol, const int nlev) { - return sizeof(Real) * (Buffer::num_2d_mid * ncol * nlev + - Buffer::num_2d_iface * ncol * (nlev + 1)); + return sizeof(Real) * + (Buffer::num_2d_mid * ncol * nlev + + Buffer::num_2d_iface * ncol * (nlev+1)); } // ON HOST, initializeÑ• the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. inline size_t init_buffer(const ATMBufferManager &buffer_manager, - const int ncol, const int nlev, Buffer &buffer) { - Real *mem = reinterpret_cast(buffer_manager.get_memory()); + const int ncol, const int nlev, + Buffer &buffer) { + Real* mem = reinterpret_cast(buffer_manager.get_memory()); // set view pointers for midpoint fields - uview_2d *view_2d_mid_ptrs[Buffer::num_2d_mid] = { - &buffer.z_mid, &buffer.dz, &buffer.qv_dry, &buffer.qc_dry, &buffer.nc_dry, - &buffer.qi_dry, &buffer.ni_dry, &buffer.w_updraft, - - // aerosol modes - &buffer.dry_int_aero_nmr[0], &buffer.dry_int_aero_nmr[1], - &buffer.dry_int_aero_nmr[2], &buffer.dry_int_aero_nmr[3], - &buffer.dry_cld_aero_nmr[0], &buffer.dry_cld_aero_nmr[1], - &buffer.dry_cld_aero_nmr[2], &buffer.dry_cld_aero_nmr[3], - - // the following requires knowledge of mam4's mode-species layout - // (see mode_aero_species() in mam4xx/aero_modes.hpp) - - // accumulation mode - &buffer.dry_int_aero_mmr[0][0], &buffer.dry_int_aero_mmr[0][1], - &buffer.dry_int_aero_mmr[0][2], &buffer.dry_int_aero_mmr[0][3], - &buffer.dry_int_aero_mmr[0][4], &buffer.dry_int_aero_mmr[0][5], - &buffer.dry_int_aero_mmr[0][6], &buffer.dry_cld_aero_mmr[0][0], - &buffer.dry_cld_aero_mmr[0][1], &buffer.dry_cld_aero_mmr[0][2], - &buffer.dry_cld_aero_mmr[0][3], &buffer.dry_cld_aero_mmr[0][4], - &buffer.dry_cld_aero_mmr[0][5], &buffer.dry_cld_aero_mmr[0][6], - - // aitken mode - &buffer.dry_int_aero_mmr[1][0], &buffer.dry_int_aero_mmr[1][1], - &buffer.dry_int_aero_mmr[1][2], &buffer.dry_int_aero_mmr[1][3], - &buffer.dry_cld_aero_mmr[1][0], &buffer.dry_cld_aero_mmr[1][1], - &buffer.dry_cld_aero_mmr[1][2], &buffer.dry_cld_aero_mmr[1][3], - - // coarse mode - &buffer.dry_int_aero_mmr[2][0], &buffer.dry_int_aero_mmr[2][1], - &buffer.dry_int_aero_mmr[2][2], &buffer.dry_int_aero_mmr[2][3], - &buffer.dry_int_aero_mmr[2][4], &buffer.dry_int_aero_mmr[2][5], - &buffer.dry_int_aero_mmr[2][6], &buffer.dry_cld_aero_mmr[2][0], - &buffer.dry_cld_aero_mmr[2][1], &buffer.dry_cld_aero_mmr[2][2], - &buffer.dry_cld_aero_mmr[2][3], &buffer.dry_cld_aero_mmr[2][4], - &buffer.dry_cld_aero_mmr[2][5], &buffer.dry_cld_aero_mmr[2][6], - - // primary carbon mode - &buffer.dry_int_aero_mmr[3][0], &buffer.dry_int_aero_mmr[3][1], - &buffer.dry_int_aero_mmr[3][2], &buffer.dry_cld_aero_mmr[3][0], - &buffer.dry_cld_aero_mmr[3][1], &buffer.dry_cld_aero_mmr[3][2], - - // aerosol gases - &buffer.dry_gas_mmr[0], &buffer.dry_gas_mmr[1], &buffer.dry_gas_mmr[2], - &buffer.dry_gas_mmr[3], &buffer.dry_gas_mmr[4], &buffer.dry_gas_mmr[5]}; - for(int i = 0; i < Buffer::num_2d_scratch; ++i) { - view_2d_mid_ptrs[Buffer::num_2d_mid + i - Buffer::num_2d_scratch] = - &buffer.scratch[i]; + uview_2d* view_2d_mid_ptrs[Buffer::num_2d_mid] = { + &buffer.z_mid, + &buffer.dz, + &buffer.qv_dry, + &buffer.qc_dry, + &buffer.nc_dry, + &buffer.qi_dry, + &buffer.ni_dry, + &buffer.w_updraft, + + // aerosol modes + &buffer.dry_int_aero_nmr[0], + &buffer.dry_int_aero_nmr[1], + &buffer.dry_int_aero_nmr[2], + &buffer.dry_int_aero_nmr[3], + &buffer.dry_cld_aero_nmr[0], + &buffer.dry_cld_aero_nmr[1], + &buffer.dry_cld_aero_nmr[2], + &buffer.dry_cld_aero_nmr[3], + + // the following requires knowledge of mam4's mode-species layout + // (see mode_aero_species() in mam4xx/aero_modes.hpp) + + // accumulation mode + &buffer.dry_int_aero_mmr[0][0], + &buffer.dry_int_aero_mmr[0][1], + &buffer.dry_int_aero_mmr[0][2], + &buffer.dry_int_aero_mmr[0][3], + &buffer.dry_int_aero_mmr[0][4], + &buffer.dry_int_aero_mmr[0][5], + &buffer.dry_int_aero_mmr[0][6], + &buffer.dry_cld_aero_mmr[0][0], + &buffer.dry_cld_aero_mmr[0][1], + &buffer.dry_cld_aero_mmr[0][2], + &buffer.dry_cld_aero_mmr[0][3], + &buffer.dry_cld_aero_mmr[0][4], + &buffer.dry_cld_aero_mmr[0][5], + &buffer.dry_cld_aero_mmr[0][6], + + // aitken mode + &buffer.dry_int_aero_mmr[1][0], + &buffer.dry_int_aero_mmr[1][1], + &buffer.dry_int_aero_mmr[1][2], + &buffer.dry_int_aero_mmr[1][3], + &buffer.dry_cld_aero_mmr[1][0], + &buffer.dry_cld_aero_mmr[1][1], + &buffer.dry_cld_aero_mmr[1][2], + &buffer.dry_cld_aero_mmr[1][3], + + // coarse mode + &buffer.dry_int_aero_mmr[2][0], + &buffer.dry_int_aero_mmr[2][1], + &buffer.dry_int_aero_mmr[2][2], + &buffer.dry_int_aero_mmr[2][3], + &buffer.dry_int_aero_mmr[2][4], + &buffer.dry_int_aero_mmr[2][5], + &buffer.dry_int_aero_mmr[2][6], + &buffer.dry_cld_aero_mmr[2][0], + &buffer.dry_cld_aero_mmr[2][1], + &buffer.dry_cld_aero_mmr[2][2], + &buffer.dry_cld_aero_mmr[2][3], + &buffer.dry_cld_aero_mmr[2][4], + &buffer.dry_cld_aero_mmr[2][5], + &buffer.dry_cld_aero_mmr[2][6], + + // primary carbon mode + &buffer.dry_int_aero_mmr[3][0], + &buffer.dry_int_aero_mmr[3][1], + &buffer.dry_int_aero_mmr[3][2], + &buffer.dry_cld_aero_mmr[3][0], + &buffer.dry_cld_aero_mmr[3][1], + &buffer.dry_cld_aero_mmr[3][2], + + // aerosol gases + &buffer.dry_gas_mmr[0], + &buffer.dry_gas_mmr[1], + &buffer.dry_gas_mmr[2], + &buffer.dry_gas_mmr[3], + &buffer.dry_gas_mmr[4], + &buffer.dry_gas_mmr[5] + }; + for (int i = 0; i < Buffer::num_2d_scratch; ++i) { + view_2d_mid_ptrs[Buffer::num_2d_mid+i-Buffer::num_2d_scratch] = &buffer.scratch[i]; } - for(int i = 0; i < Buffer::num_2d_mid; ++i) { + for (int i = 0; i < Buffer::num_2d_mid; ++i) { *view_2d_mid_ptrs[i] = view_2d(mem, ncol, nlev); mem += view_2d_mid_ptrs[i]->size(); } // set view pointers for interface fields - uview_2d *view_2d_iface_ptrs[Buffer::num_2d_iface] = {&buffer.z_iface}; - for(int i = 0; i < Buffer::num_2d_iface; ++i) { - *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev + 1); + uview_2d* view_2d_iface_ptrs[Buffer::num_2d_iface] = { + &buffer.z_iface + }; + for (int i = 0; i < Buffer::num_2d_iface; ++i) { + *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev+1); mem += view_2d_iface_ptrs[i]->size(); } @@ -451,48 +509,47 @@ inline size_t init_buffer(const ATMBufferManager &buffer_manager, /* // Compute workspace manager size to check used memory vs. requested memory // (if needed) - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); const int n_wind_slots = ekat::npack(2)*Spack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, - 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); mem += wsm_size; + const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); + mem += wsm_size; */ // return the number of bytes allocated - return (mem - buffer_manager.get_memory()) * sizeof(Real); + return (mem - buffer_manager.get_memory())*sizeof(Real); } // Given a dry atmosphere state, creates a haero::Atmosphere object for the // column with the given index. This object can be provided to mam4xx for the // column. KOKKOS_INLINE_FUNCTION -haero::Atmosphere atmosphere_for_column(const DryAtmosphere &dry_atm, +haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, const int column_index) { EKAT_KERNEL_ASSERT_MSG(dry_atm.T_mid.data() != nullptr, - "T_mid not defined for dry atmosphere state!"); + "T_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_mid.data() != nullptr, - "p_mid not defined for dry atmosphere state!"); + "p_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qv.data() != nullptr, - "qv not defined for dry atmosphere state!"); + "qv not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qc.data() != nullptr, - "qc not defined for dry atmosphere state!"); + "qc not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.nc.data() != nullptr, - "nc not defined for dry atmosphere state!"); + "nc not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qi.data() != nullptr, - "qi not defined for dry atmosphere state!"); + "qi not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.ni.data() != nullptr, - "ni not defined for dry atmosphere state!"); + "ni not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.z_mid.data() != nullptr, - "z_mid not defined for dry atmosphere state!"); + "z_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del.data() != nullptr, - "p_del not defined for dry atmosphere state!"); + "p_del not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_int.data() != nullptr, - "p_int not defined for dry atmosphere state!"); + "p_int not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.cldfrac.data() != nullptr, - "cldfrac not defined for dry atmosphere state!"); + "cldfrac not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, - "w_updraft not defined for dry atmosphere state!"); + "w_updraft not defined for dry atmosphere state!"); return haero::Atmosphere(mam4::nlev, ekat::subview(dry_atm.T_mid, column_index), ekat::subview(dry_atm.p_mid, column_index), @@ -509,29 +566,38 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere &dry_atm, dry_atm.pblh(column_index)); } +KOKKOS_INLINE_FUNCTION +int get_haero_prognostics_index(const int mode, const int species) +{ + const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); + const int ind = + (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; + return ind; +}; + // Given an AerosolState with views for dry aerosol quantities, creates a // mam4::Prognostics object for the column with the given index with // ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to // mam4xx for the column. KOKKOS_INLINE_FUNCTION -mam4::Prognostics interstitial_aerosols_for_column(const AerosolState &dry_aero, +mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, const int column_index) { constexpr int nlev = mam4::nlev; mam4::Prognostics progs(nlev); - for(int m = 0; m < num_aero_modes(); ++m) { + for (int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), - "int_aero_nmr not defined for dry aerosol state!"); + "int_aero_nmr not defined for dry aerosol state!"); progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); - for(int a = 0; a < num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - progs.q_aero_i[m][a] = - ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + int hindex = get_haero_prognostics_index(m,a); + progs.q_aero_i[m][hindex] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); } } } - for(int g = 0; g < num_aero_gases(); ++g) { + for (int g = 0; g < num_aero_gases(); ++g) { EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), - "gas_mmr not defined for dry aerosol state!"); + "gas_mmr not defined for dry aerosol state!"); progs.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); } return progs; @@ -541,17 +607,17 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState &dry_aero, // with the given index with interstitial and cloudborne aerosol views defined. // This object can be provided to mam4xx for the column. KOKKOS_INLINE_FUNCTION -mam4::Prognostics aerosols_for_column(const AerosolState &dry_aero, +mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, const int column_index) { auto progs = interstitial_aerosols_for_column(dry_aero, column_index); - for(int m = 0; m < num_aero_modes(); ++m) { + for (int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), - "dry_cld_aero_nmr not defined for aerosol state!"); + "dry_cld_aero_nmr not defined for aerosol state!"); progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); - for(int a = 0; a < num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - progs.q_aero_c[m][a] = - ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.cld_aero_mmr[m][a].data()) { + int hindex = get_haero_prognostics_index(m,a); + progs.q_aero_c[m][hindex] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); } } } @@ -577,27 +643,26 @@ void compute_vertical_layer_heights(const Team& team, }*/ KOKKOS_INLINE_FUNCTION -void compute_vertical_layer_heights(const Team &team, - const DryAtmosphere &dry_atm, +void compute_vertical_layer_heights(const Team& team, + const DryAtmosphere& dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); - const auto dz = ekat::subview(dry_atm.dz, column_index); - const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine + const auto dz = ekat::subview(dry_atm.dz, column_index); + const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); - const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); - const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); - const auto qv = ekat::subview(dry_atm.qv, column_index); + const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); + const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); - PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs - dz); // output + PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs + dz);//output team.team_barrier(); - PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, // inputs - z_iface); // output - team.team_barrier(); // likely necessary to have z_iface up to date + PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs + z_iface); //output + team.team_barrier(); // likely necessary to have z_iface up to date PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } @@ -605,23 +670,19 @@ void compute_vertical_layer_heights(const Team &team, // team to compute the vertical updraft velocity for the column with the given // index. KOKKOS_INLINE_FUNCTION -void compute_updraft_velocities(const Team &team, const WetAtmosphere &wet_atm, - const DryAtmosphere &dry_atm, +void compute_updraft_velocities(const Team& team, + const WetAtmosphere& wet_atm, + const DryAtmosphere& dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { - dry_atm.dz(i, k) = - PF::calculate_dz(dry_atm.p_del(i, k), dry_atm.p_mid(i, k), - dry_atm.T_mid(i, k), wet_atm.qv(i, k)); - const auto rho = - PF::calculate_density(dry_atm.p_del(i, k), dry_atm.dz(i, k)); - dry_atm.w_updraft(i, k) = - PF::calculate_vertical_velocity(wet_atm.omega(i, k), rho); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + dry_atm.dz(i,k) = PF::calculate_dz(dry_atm.p_del(i,k), dry_atm.p_mid(i,k), dry_atm.T_mid(i,k), wet_atm.qv(i,k)); + const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); + dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); }); } @@ -629,108 +690,93 @@ void compute_updraft_velocities(const Team &team, const WetAtmosphere &wet_atm, // from the team to compute mixing ratios for a dry atmosphere state in th // column with the given index. KOKKOS_INLINE_FUNCTION -void compute_dry_mixing_ratios(const Team &team, const WetAtmosphere &wet_atm, - const DryAtmosphere &dry_atm, +void compute_dry_mixing_ratios(const Team& team, + const WetAtmosphere& wet_atm, + const DryAtmosphere& dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { - const auto qv_ik = wet_atm.qv(i, k); - dry_atm.qv(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i, k), qv_ik); - dry_atm.qc(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i, k), qv_ik); - dry_atm.nc(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i, k), qv_ik); - dry_atm.qi(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i, k), qv_ik); - dry_atm.ni(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i, k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = wet_atm.qv(i,k); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } -// Given a thread team and wet atmospheric and aerosol states, dispatches -// threads from the team to compute mixing ratios for the given dry interstitial -// aerosol state for the column with the given index. +// Given a thread team and wet atmospheric and aerosol states, dispatches threads +// from the team to compute mixing ratios for the given dry interstitial aerosol +// state for the column with the given index. KOKKOS_INLINE_FUNCTION -void compute_dry_mixing_ratios(const Team &team, const WetAtmosphere &wet_atm, - const AerosolState &wet_aero, - const AerosolState &dry_aero, +void compute_dry_mixing_ratios(const Team& team, + const WetAtmosphere& wet_atm, + const AerosolState& wet_aero, + const AerosolState& dry_aero, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { - const auto qv_ik = wet_atm.qv(i, k); - for(int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i, k) = PF::calculate_drymmr_from_wetmmr( - wet_aero.int_aero_nmr[m](i, k), qv_ik); - if(dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i, k) = PF::calculate_drymmr_from_wetmmr( - wet_aero.cld_aero_nmr[m](i, k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = wet_atm.qv(i,k); + for (int m = 0; m < num_aero_modes(); ++m) { + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + if (dry_aero.cld_aero_nmr[m].data()) { + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } - for(int a = 0; a < num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i, k) = PF::calculate_drymmr_from_wetmmr( - wet_aero.int_aero_mmr[m][a](i, k), qv_ik); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } - if(dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i, k) = PF::calculate_drymmr_from_wetmmr( - wet_aero.cld_aero_mmr[m][a](i, k), qv_ik); + if (dry_aero.cld_aero_mmr[m][a].data()) { + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } - for(int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i, k) = - PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i, k), qv_ik); + for (int g = 0; g < num_aero_gases(); ++g) { + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } -// Given a thread team and dry atmospheric and aerosol states, dispatches -// threads from the team to compute mixing ratios for the given wet interstitial -// aerosol state for the column with the given index. +// Given a thread team and dry atmospheric and aerosol states, dispatches threads +// from the team to compute mixing ratios for the given wet interstitial aerosol +// state for the column with the given index. KOKKOS_INLINE_FUNCTION -void compute_wet_mixing_ratios(const Team &team, const DryAtmosphere &dry_atm, - const AerosolState &dry_aero, - const AerosolState &wet_aero, +void compute_wet_mixing_ratios(const Team& team, + const DryAtmosphere& dry_atm, + const AerosolState& dry_aero, + const AerosolState& wet_aero, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { - const auto qv_ik = dry_atm.qv(i, k); - for(int m = 0; m < num_aero_modes(); ++m) { - wet_aero.int_aero_nmr[m](i, k) = PF::calculate_wetmmr_from_drymmr( - dry_aero.int_aero_nmr[m](i, k), qv_ik); - if(wet_aero.cld_aero_nmr[m].data()) { - wet_aero.cld_aero_nmr[m](i, k) = PF::calculate_wetmmr_from_drymmr( - dry_aero.cld_aero_nmr[m](i, k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = dry_atm.qv(i,k); + for (int m = 0; m < num_aero_modes(); ++m) { + wet_aero.int_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_nmr[m](i,k), qv_ik); + if (wet_aero.cld_aero_nmr[m].data()) { + wet_aero.cld_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_nmr[m](i,k), qv_ik); } - for(int a = 0; a < num_aero_species(); ++a) { - if(wet_aero.int_aero_mmr[m][a].data()) { - wet_aero.int_aero_mmr[m][a](i, k) = PF::calculate_wetmmr_from_drymmr( - dry_aero.int_aero_mmr[m][a](i, k), qv_ik); + for (int a = 0; a < num_aero_species(); ++a) { + if (wet_aero.int_aero_mmr[m][a].data()) { + wet_aero.int_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_mmr[m][a](i,k), qv_ik); } - if(wet_aero.cld_aero_mmr[m][a].data()) { - wet_aero.cld_aero_mmr[m][a](i, k) = PF::calculate_wetmmr_from_drymmr( - dry_aero.cld_aero_mmr[m][a](i, k), qv_ik); + if (wet_aero.cld_aero_mmr[m][a].data()) { + wet_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } - for(int g = 0; g < num_aero_gases(); ++g) { - wet_aero.gas_mmr[g](i, k) = - PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i, k), qv_ik); + for (int g = 0; g < num_aero_gases(); ++g) { + wet_aero.gas_mmr[g](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i,k), qv_ik); } }); } @@ -740,55 +786,50 @@ void compute_wet_mixing_ratios(const Team &team, const DryAtmosphere &dry_atm, // these constants where needed within two such functions so we don't define // them inconsistently. Yes, it's the 21st century and we're still struggling // with these basic things. -#define DECLARE_PROG_TRANSFER_CONSTANTS \ - /* mapping of constituent indices to aerosol modes */ \ - const auto Accum = mam4::ModeIndex::Accumulation; \ - const auto Aitken = mam4::ModeIndex::Aitken; \ - const auto Coarse = mam4::ModeIndex::Coarse; \ - const auto PC = mam4::ModeIndex::PrimaryCarbon; \ - const auto NoMode = mam4::ModeIndex::None; \ - static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) \ - */ \ - Accum, Accum, Accum, Accum, Accum, Accum, Accum, \ - Accum, /* 7 aero species + NMR */ \ - Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ - Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, \ - Coarse, /* 7 aero species + NMR */ \ - PC, PC, PC, PC, /* 3 aero species + NMR */ \ - }; \ - /* mapping of constituent indices to aerosol species */ \ - const auto SOA = mam4::AeroId::SOA; \ - const auto SO4 = mam4::AeroId::SO4; \ - const auto POM = mam4::AeroId::POM; \ - const auto BC = mam4::AeroId::BC; \ - const auto NaCl = mam4::AeroId::NaCl; \ - const auto DST = mam4::AeroId::DST; \ - const auto MOM = mam4::AeroId::MOM; \ - const auto NoAero = mam4::AeroId::None; \ - static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ - NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) \ - */ \ - SO4, POM, SOA, BC, DST, NaCl, MOM, \ - NoAero, /* accumulation mode */ \ - SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ - DST, NaCl, SO4, BC, POM, SOA, MOM, \ - NoAero, /* coarse mode */ \ - POM, BC, MOM, NoAero, /* primary carbon mode */ \ - }; \ - /* mapping of constituent indices to gases */ \ - const auto O3 = mam4::GasId::O3; \ - const auto H2O2 = mam4::GasId::H2O2; \ - const auto H2SO4 = mam4::GasId::H2SO4; \ - const auto SO2 = mam4::GasId::SO2; \ - const auto DMS = mam4::GasId::DMS; \ - const auto SOAG = mam4::GasId::SOAG; \ - const auto NoGas = mam4::GasId::None; \ - static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ - O3, H2O2, H2SO4, SO2, DMS, SOAG, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ +#define DECLARE_PROG_TRANSFER_CONSTANTS \ + /* mapping of constituent indices to aerosol modes */ \ + const auto Accum = mam4::ModeIndex::Accumulation; \ + const auto Aitken = mam4::ModeIndex::Aitken; \ + const auto Coarse = mam4::ModeIndex::Coarse; \ + const auto PC = mam4::ModeIndex::PrimaryCarbon; \ + const auto NoMode = mam4::ModeIndex::None; \ + static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ + Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ + Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ + Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ + PC, PC, PC, PC, /* 3 aero species + NMR */ \ + }; \ + /* mapping of constituent indices to aerosol species */ \ + const auto SOA = mam4::AeroId::SOA; \ + const auto SO4 = mam4::AeroId::SO4; \ + const auto POM = mam4::AeroId::POM; \ + const auto BC = mam4::AeroId::BC; \ + const auto NaCl = mam4::AeroId::NaCl; \ + const auto DST = mam4::AeroId::DST; \ + const auto MOM = mam4::AeroId::MOM; \ + const auto NoAero = mam4::AeroId::None; \ + static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ + NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) */ \ + SO4, POM, SOA, BC, DST, NaCl, MOM, NoAero, /* accumulation mode */ \ + SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ + DST, NaCl, SO4, BC, POM, SOA, MOM, NoAero, /* coarse mode */ \ + POM, BC, MOM, NoAero, /* primary carbon mode */ \ + }; \ + /* mapping of constituent indices to gases */ \ + const auto O3 = mam4::GasId::O3; \ + const auto H2O2 = mam4::GasId::H2O2; \ + const auto H2SO4 = mam4::GasId::H2SO4; \ + const auto SO2 = mam4::GasId::SO2; \ + const auto DMS = mam4::GasId::DMS; \ + const auto SOAG = mam4::GasId::SOAG; \ + const auto NoGas = mam4::GasId::None; \ + static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ + O3, H2O2, H2SO4, SO2, DMS, SOAG, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, \ }; // Given a Prognostics object, transfers data for interstitial aerosols to the @@ -801,29 +842,30 @@ void compute_wet_mixing_ratios(const Team &team, const DryAtmosphere &dry_atm, // NOTE: indices KOKKOS_INLINE_FUNCTION void transfer_prognostics_to_work_arrays(const mam4::Prognostics &progs, - const int k, Real q[gas_pcnst()], + const int k, + Real q[gas_pcnst()], Real qqcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS // copy number/mass mixing ratios from progs to q and qqcw at level k, // converting them to VMR - for(int i = 0; i < gas_pcnst(); ++i) { + for (int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if(gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); - q[i] = progs.q_gas[g](k); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + q[i] = progs.q_gas[g](k); qqcw[i] = progs.q_gas[g](k); } else { int m = static_cast(mode_index); - if(aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); - q[i] = progs.q_aero_i[m][a](k); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + q[i] = progs.q_aero_i[m][a](k); qqcw[i] = progs.q_aero_c[m][a](k); - } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); - q[i] = progs.n_mode_i[m](k); + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); + q[i] = progs.n_mode_i[m](k); qqcw[i] = progs.n_mode_c[m](k); } } @@ -839,29 +881,34 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], Real vmrcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS - for(int i = 0; i < gas_pcnst(); ++i) { + for (int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if(gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); const Real mw = mam4::gas_species(g).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { <<<<<<< HEAD if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); ======= int m = static_cast(mode_index); +<<<<<<< HEAD if(aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); >>>>>>> Modified species order to match E3SM with the folling change in mam4xx +======= + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const Real mw = mam4::aero_species(a).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); - } else { // constituent is a modal number mixing ratio - vmr[i] = q[i]; + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + } else { // constituent is a modal number mixing ratio + vmr[i] = q[i]; vmrcw[i] = qqcw[i]; } } @@ -873,32 +920,38 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], KOKKOS_INLINE_FUNCTION void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], const Real vmrcw[gas_pcnst()], - Real q[gas_pcnst()], Real qqcw[gas_pcnst()]) { + Real q[gas_pcnst()], + Real qqcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS - for(int i = 0; i < gas_pcnst(); ++i) { + for (int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if(gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); const Real mw = mam4::gas_species(g).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { <<<<<<< HEAD if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); ======= int m = static_cast(mode_index); +<<<<<<< HEAD if(aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); >>>>>>> Modified species order to match E3SM with the folling change in mam4xx +======= + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const Real mw = mam4::aero_species(a).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); - } else { // constituent is a modal number mixing ratio - q[i] = vmr[i]; + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + } else { // constituent is a modal number mixing ratio + q[i] = vmr[i]; qqcw[i] = vmrcw[i]; } } @@ -911,31 +964,35 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], KOKKOS_INLINE_FUNCTION void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], const Real qqcw[gas_pcnst()], - mam4::Prognostics &progs, - const int k) { + mam4::Prognostics &progs, const int k) { DECLARE_PROG_TRANSFER_CONSTANTS // copy number/mass mixing ratios from progs to q and qqcw at level k, // converting them to VMR - for(int i = 0; i < gas_pcnst(); ++i) { + for (int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if(gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); progs.q_gas[g](k) = q[i]; } else { int m = static_cast(mode_index); - if(aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); progs.q_aero_i[m][a](k) = q[i]; progs.q_aero_c[m][a](k) = qqcw[i]; +<<<<<<< HEAD <<<<<<< HEAD } else { // constituent is a modal number mixing ratio ======= } else { // constituent is a modal number mixing ratio int m = static_cast(mode_index); >>>>>>> Modified species order to match E3SM with the folling change in mam4xx +======= + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index progs.n_mode_i[m](k) = q[i]; progs.n_mode_c[m](k) = qqcw[i]; } @@ -945,6 +1002,6 @@ void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], #undef DECLARE_PROG_TRANSFER_CONSTANTS -} // namespace scream::mam_coupling +} // namespace scream::mam_coupling #endif From a1d289bd7bfccfd994d07f75b1e8f2970fe4c562 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 25 Mar 2024 13:40:12 -0700 Subject: [PATCH 042/142] Passes dt as an arg to make hetro freeze work --- .../mam/eamxx_mam_aci_process_interface.cpp | 225 +++++++++--------- .../eamxx/src/physics/mam/mam_coupling.hpp | 1 + 2 files changed, 111 insertions(+), 115 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 2e32e067077..285beec7d42 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -591,8 +591,8 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, - MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], - const int nlev) { + MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], const int nlev, + const double dt) { using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); @@ -608,107 +608,106 @@ void call_hetfrz_compute_tendencies( MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - // Kokkos::parallel_for( - // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - // const int icol = team.league_rank(); - for(int icol = 0; icol < 1; icol++) { - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies - // class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, - ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, dummy, - dummy, dummy, dummy, dummy, pblh); - // set surface state data - haero::Surface surf{}; - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); - - const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - - // BALLI - mam4::Diagnostics diags(nlev); - diags.stratiform_cloud_fraction = - ekat::subview(stratiform_cloud_fraction, icol); - diags.activation_fraction[accum_idx] = - ekat::subview(activation_fraction_accum_idx, icol); - diags.activation_fraction[coarse_idx] = - ekat::subview(activation_fraction_coarse_idx, icol); - - // These are the output tendencies from heterogeneous freezing that need - // to be added correctly to the cloud-micorphysics scheme. - diags.hetfrz_immersion_nucleation_tend = - ekat::subview(hetfrz_immersion_nucleation_tend, icol); - diags.hetfrz_contact_nucleation_tend = - ekat::subview(hetfrz_contact_nucleation_tend, icol); - diags.hetfrz_depostion_nucleation_tend = - ekat::subview(hetfrz_depostion_nucleation_tend, icol); - - diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); - diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); - diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); - diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); - diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); - diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); - diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); - diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); - diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); - diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); - diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); - diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); - diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); - diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); - diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); - diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); - diags.na500 = ekat::subview(diagnostic_scratch[18], icol); - diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); - diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); - diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); - diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); - diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); - diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); - diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); - diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); - diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); - diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); - diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); - diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); - diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); - diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); - diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); - diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); - diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); - diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); - diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); - diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); - diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); - diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - // diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - /*diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - //------------------------------------------------------------- - // Heterogeneous freezing - // frzimm, frzcnt, frzdep are the outputs of - // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) - //------------------------------------------------------------- - // - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t = 0, dt = 0;*/ - // hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, - // progs, - // diags, tends); - // }); - } + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // for(int icol = 0; icol < 1; icol++) { + // Set up an atmosphere, surface, diagnostics, pronostics and + // tendencies class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, + ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, + dummy, dummy, dummy, dummy, dummy, pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + + // BALLI + mam4::Diagnostics diags(nlev); + diags.stratiform_cloud_fraction = + ekat::subview(stratiform_cloud_fraction, icol); + diags.activation_fraction[accum_idx] = + ekat::subview(activation_fraction_accum_idx, icol); + diags.activation_fraction[coarse_idx] = + ekat::subview(activation_fraction_coarse_idx, icol); + + // These are the output tendencies from heterogeneous freezing that need + // to be added correctly to the cloud-micorphysics scheme. + diags.hetfrz_immersion_nucleation_tend = + ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = + ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = + ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0; //, dt = 0; + hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, + progs, diags, tends); + }); + //} } } // namespace @@ -1176,16 +1175,13 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for for tke_ to be computed. - compute_subgrid_scale_velocities(team_policy, wsub_ /*output*/, - wsubice_ /*output*/, wsig_ /*output*/, tke_, + compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, wsubmin, top_lev_, nlev_); Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w2_ /*output*/, w0_, - wsig_, nlev_); + compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); - compute_aitken_dry_diameter(team_policy, aitken_dry_dia_ /*output*/, dgnum_, - top_lev_); + compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. // FIXME: Find out in-outs of the following call! @@ -1193,9 +1189,8 @@ void MAMAci::run_impl(const double dt) { nidep_, nimey_, naai_hom_, naai_, dry_aero_, dry_atmosphere_, aitken_dry_dia_, nlev_); - store_liquid_cloud_fraction(team_policy, cloud_frac_new_ /*output*/, - cloud_frac_old_ /*output*/, wet_atmosphere_, - liqcldf_, top_lev_); + store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, + wet_atmosphere_, liqcldf_, top_lev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- @@ -1214,8 +1209,8 @@ void MAMAci::run_impl(const double dt) { rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_ /*output*/, - dry_atmosphere_.p_del, nlev_); + compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, + nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. call_function_dropmixnuc( @@ -1239,7 +1234,7 @@ void MAMAci::run_impl(const double dt) { stratiform_cloud_fraction_, activation_fraction_accum_idx_, activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, - naai_hom_, naai_, diagnostic_scratch_, nlev_); + naai_hom_, naai_, diagnostic_scratch_, nlev_, dt); Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index a69bd9f2042..21408fa7b5b 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -566,6 +566,7 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, dry_atm.pblh(column_index)); } +//FIXME: move it to mam4xx and add comments and change variable names KOKKOS_INLINE_FUNCTION int get_haero_prognostics_index(const int mode, const int species) { From f958170ea69aeeedc385a5e0adafc6c033e9ce5d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 25 Mar 2024 14:38:29 -0700 Subject: [PATCH 043/142] Codes works after changes in mam4xx hetfrz.hpp were reverted to original --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 285beec7d42..fa6700293de 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -704,8 +704,8 @@ void call_hetfrz_compute_tendencies( const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t = 0; //, dt = 0; - hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, - progs, diags, tends); + hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, + diags, tends); }); //} } From 2ab183f50cd44f0f655c4662cdbcca9eb81ef062 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 25 Mar 2024 14:53:58 -0700 Subject: [PATCH 044/142] Moves function get_haero_prognostics_index to mam4xx --- .../eamxx/src/physics/mam/mam_coupling.hpp | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 21408fa7b5b..a80404757bd 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -566,16 +566,6 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, dry_atm.pblh(column_index)); } -//FIXME: move it to mam4xx and add comments and change variable names -KOKKOS_INLINE_FUNCTION -int get_haero_prognostics_index(const int mode, const int species) -{ - const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); - const int ind = - (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; - return ind; -}; - // Given an AerosolState with views for dry aerosol quantities, creates a // mam4::Prognostics object for the column with the given index with // ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to @@ -591,8 +581,8 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - int hindex = get_haero_prognostics_index(m,a); - progs.q_aero_i[m][hindex] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + int h_ind = mam4::utils::get_haero_prognostics_index(m,a);//Index of HAERO's prognostics + progs.q_aero_i[m][h_ind] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); } } } @@ -617,8 +607,8 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.cld_aero_mmr[m][a].data()) { - int hindex = get_haero_prognostics_index(m,a); - progs.q_aero_c[m][hindex] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + int h_ind = mam4::utils::get_haero_prognostics_index(m,a); //Index of HAERO's prognostics array + progs.q_aero_c[m][h_ind] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); } } } From cbe943daf5f13523073e4748f83a4f8042f0a78c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 11:36:48 -0700 Subject: [PATCH 045/142] Some minor FIXME comments and cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 3 ++- .../eamxx/src/physics/mam/mam_coupling.hpp | 16 +--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index fa6700293de..ba77611fcac 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1066,10 +1066,11 @@ void MAMAci::initialize_impl(const RunType run_type) { get_field_out("activation_fraction_coarse").get_view(); // allocate work - + // FIXME: store 25 into a const var or get from MAM4xx for(int icnst = 0; icnst < 25; ++icnst) { qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); } + // FIXME :store 40 in a const int or get from MAM4xx state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, 40); // interstitial and cloudborne aerosol tracers of interest: mass (q) and diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index a80404757bd..6463ff73e8b 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -618,21 +618,6 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, // Given a thread team and a dry atmosphere state, dispatches threads from the // team to compute vertical layer heights and interfaces for the column with // the given index. -/*KOKKOS_INLINE_FUNCTION -void compute_vertical_layer_heights(const Team& team, - const DryAtmosphere& dry_atm, - const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); - - const auto dz = ekat::subview(dry_atm.dz, column_index); - auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - auto z_mid = ekat::subview(dry_atm.z_mid, column_index); - PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, z_iface); - team.team_barrier(); // likely necessary to have z_iface up to date - PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); -}*/ - KOKKOS_INLINE_FUNCTION void compute_vertical_layer_heights(const Team& team, const DryAtmosphere& dry_atm, @@ -657,6 +642,7 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } + // Given a thread team and wet and dry atmospheres, dispatches threads from the // team to compute the vertical updraft velocity for the column with the given // index. From 3ffe00602cb35e42f84a8368f2957f2ba39e8a83 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 16:23:50 -0700 Subject: [PATCH 046/142] Another round of minor cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 4 --- .../mam/eamxx_mam_aci_process_interface.hpp | 34 ++++++++++++------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ba77611fcac..4552a7d8bc8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,9 +1,5 @@ #include -#include "ekat/util/ekat_units.hpp" -#include "mam4xx/aero_config.hpp" -#include "mam4xx/ndrop.hpp" - namespace scream { namespace { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 88d91b999cd..a70ac5a6a6a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -7,8 +7,14 @@ // For declaring ACI class derived from atm process class #include -// For physical constants -#include "physics/share/physics_constants.hpp" +// For EKAT units package +#include "ekat/util/ekat_units.hpp" + +// For aerosol configuration +#include "mam4xx/aero_config.hpp" + +// For calling ndrop functions +#include "mam4xx/ndrop.hpp" namespace scream { @@ -19,16 +25,11 @@ class MAMAci final : public scream::AtmosphereProcess { mam4::Hetfrz hetfrz_; // views for single- and multi-column data - using view_1d = scream::mam_coupling::view_1d; using view_2d = scream::mam_coupling::view_2d; using view_3d = scream::mam_coupling::view_3d; - using const_view_1d = scream::mam_coupling::const_view_1d; using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; - template - using view_4d = KT::view; - // FIXME:B: Should the following variables be public? They are like that in // micriphysics and optics codes // FIXME the time step for microphysics [s] need to get from the input @@ -43,16 +44,20 @@ class MAMAci final : public scream::AtmosphereProcess { // turbulent kinetic energy [m^2/s^2] view_2d tke_; + // Dry diameter of the aitken mode (for ice nucleation) view_2d aitken_dry_dia_; - view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()] - [mam_coupling::num_aero_species()]; - + // wet mixing ratios (water species) mam_coupling::WetAtmosphere wet_atmosphere_; + // dry mixing ratios (water species) mam_coupling::DryAtmosphere dry_atmosphere_; + // aerosol dry diameter const_view_3d dgnum_; + + // FIXME: Make sure these output are used somewhere and add comments about + // these view_2d nihf_; view_2d niim_; view_2d nidep_; @@ -128,20 +133,22 @@ class MAMAci final : public scream::AtmosphereProcess { std::shared_ptr grid_; // A view array to carry cloud borne aerosol mmrs/nmrs + // FIXME: 25 should be a const int view_2d qqcw_fld_work_[25]; + + // A view to carry interstitial aerosol mmrs/nmrs view_3d state_q_work_; public: // Constructor MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); - // process metadata - // Return type of the process + // Process metadata: Return type of the process AtmosphereProcessType MAMAci::type() const { return AtmosphereProcessType::Physics; } - // return name of the process + // Return name of the process std::string MAMAci::name() const { return "mam4_aci"; } // grid @@ -152,6 +159,7 @@ class MAMAci final : public scream::AtmosphereProcess { size_t MAMAci::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } + void init_buffers(const ATMBufferManager &buffer_manager) override; // process behavior From 2d4ecf9c352a2bfee843db6f646d876200f187d6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 18:14:12 -0700 Subject: [PATCH 047/142] Replaces dtmicro with dt and some cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 32 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 6 ++-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4552a7d8bc8..93ee66825a8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -370,7 +370,7 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, mam_coupling::DryAtmosphere &dry_atmosphere, - const mam_coupling::AerosolState &dry_aerosol_state, const Real dtmicro, + const mam_coupling::AerosolState &dry_aerosol_state, const Real dt, MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], @@ -386,6 +386,8 @@ void call_function_dropmixnuc( MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { + // FIXME: why can't we use MAMAci::drop_scratch_ above + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; MAMAci::const_view_2d zm = dry_atmosphere.z_mid; @@ -515,9 +517,9 @@ void call_function_dropmixnuc( } mam4::ndrop::dropmixnuc( - team, dtmicro, ekat::subview(T_mid, icol), - ekat::subview(p_mid, icol), ekat::subview(p_int, icol), - ekat::subview(pdel, icol), ekat::subview(rpdel, icol), + team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), + ekat::subview(p_int, icol), ekat::subview(pdel, icol), + ekat::subview(rpdel, icol), ekat::subview( zm, icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] @@ -716,17 +718,14 @@ MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) // set grid for all the inputs and outputs void MAMAci::set_grids( const std::shared_ptr grids_manager) { - m_atm_logger->log(ekat::logger::LogLevel::info, "Calling ACI set grid"); - grid_ = grids_manager->get_grid("Physics"); // Use physics grid const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - Kokkos::resize( - rho_, ncol_, - nlev_); // BQ: Does Kokkos::resize allocates memory as well for rho_? + // Kokkos::resize only works on host to allocates memory + Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -738,7 +737,7 @@ void MAMAci::set_grids( Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); Kokkos::resize(rpdel_, ncol_, nlev_); - for(int i = 0; i < 15; ++i) { + for(int i = 0; i < drop_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } // Define the different field layouts that will be used for this process @@ -751,12 +750,8 @@ void MAMAci::set_grids( // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; - ekat::units::Units kg = ekat::units::kg; // BQ: why do we need to do this??? - ekat::units::Units Pa = ekat::units::Pa; - ekat::units::Units s = ekat::units::s; - ekat::units::Units m = ekat::units::m; - ekat::units::Units K = ekat::units::K; - auto q_unit = kg / kg; // units of mass mixing ratios of tracers + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); @@ -774,6 +769,7 @@ void MAMAci::set_grids( "tracers"); // cloud ice number mixing ratio [1/kg] add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints + add_field( "omega", scalar3d_layout_mid, Pa / s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints @@ -786,6 +782,8 @@ void MAMAci::set_grids( add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height + + // Output from this process add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, @@ -1211,7 +1209,7 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. call_function_dropmixnuc( - team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, + team_policy, dry_atmosphere_, dry_aero_, dt, raercol_cw_, raercol_, qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, dry_atmosphere_.p_del, rpdel_, state_q_work_, ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index a70ac5a6a6a..403de888024 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -32,8 +32,6 @@ class MAMAci final : public scream::AtmosphereProcess { // FIXME:B: Should the following variables be public? They are like that in // micriphysics and optics codes - // FIXME the time step for microphysics [s] need to get from the input - const Real dtmicro_ = .0001; // rho is air density [kg/m3] view_2d rho_; @@ -96,7 +94,9 @@ class MAMAci final : public scream::AtmosphereProcess { view_3d nact_; view_3d mact_; - view_2d dropmixnuc_scratch_mem_[15]; + + static constexpr int drop_scratch_ = 15; + view_2d dropmixnuc_scratch_mem_[drop_scratch_]; view_2d stratiform_cloud_fraction_; view_2d activation_fraction_accum_idx_; From f7b9c112896e6d03444c33175fd3d26f97ecd6dc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 18:30:23 -0700 Subject: [PATCH 048/142] Replaces ncldwtr with dry_atmospher.nc --- .../mam/eamxx_mam_aci_process_interface.cpp | 22 +++++++------------ .../mam/eamxx_mam_aci_process_interface.hpp | 2 -- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 93ee66825a8..ee4a8ee9661 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -379,8 +379,8 @@ void call_function_dropmixnuc( MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, MAMAci::view_2d rpdel, MAMAci::view_3d state_q_work_, - MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, - MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, + MAMAci::const_view_2d nc, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, + MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, @@ -523,7 +523,7 @@ void call_function_dropmixnuc( ekat::subview( zm, icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - ekat::subview(state_q_work_loc, icol), ekat::subview(ncldwtr, icol), + ekat::subview(state_q_work_loc, icol), ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] ekat::subview(cloud_frac_new, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, @@ -791,10 +791,6 @@ void MAMAci::set_grids( add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // MUST FIXME: Is it same as nc or may be not???? - add_field("ncldwtr", scalar3d_layout_mid, n_unit, - grid_name); // initial droplet number mixing ratio [#/kg] - // MUST FIXME: This should be an internal variable. why we need this as an // input??? add_field("w_updraft", scalar3d_layout_mid, q_unit, @@ -986,9 +982,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // set atmosphere state data w_sec_ = get_field_in("w_sec").get_view(); - ncldwtr_ = - get_field_in("ncldwtr") - .get_view(); // MUST FIXME: is is nc, may be not??? + dgnum_ = get_field_in("dgnum") .get_view(); // MUST FIXME: is it an input, can // we compute it using calcsize??? @@ -1211,10 +1205,10 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc( team_policy, dry_atmosphere_, dry_aero_, dt, raercol_cw_, raercol_, qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, - dry_atmosphere_.p_del, rpdel_, state_q_work_, ncldwtr_, kvh_, qcld_, - wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, - ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, - nlev_); + dry_atmosphere_.p_del, rpdel_, state_q_work_, dry_atmosphere_.nc, kvh_, + qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, + ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, + dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 403de888024..3d30d797a82 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -65,8 +65,6 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - const_view_2d ncldwtr_; - view_2d cloud_frac_new_; view_2d cloud_frac_old_; view_2d qcld_; From eb06b3bdc42a393416b365c2ec574233b93a0f08 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 21:34:41 -0700 Subject: [PATCH 049/142] changes some variable names and add dummy comments to fix later --- .../mam/eamxx_mam_aci_process_interface.cpp | 147 ++++++++++-------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- 2 files changed, 83 insertions(+), 68 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ee4a8ee9661..da7bd10e5c7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -581,8 +581,8 @@ void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, mam_coupling::AerosolState &dry_aero_, - mam_coupling::WetAtmosphere &wet_atmosphere_, - mam_coupling::DryAtmosphere &dry_atmosphere_, + mam_coupling::WetAtmosphere &wet_atm_, + mam_coupling::DryAtmosphere &dry_atm_, MAMAci::view_2d stratiform_cloud_fraction, MAMAci::view_2d activation_fraction_accum_idx, MAMAci::view_2d activation_fraction_coarse_idx, @@ -596,8 +596,8 @@ void call_hetfrz_compute_tendencies( mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; - mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; + mam_coupling::WetAtmosphere wet_atmosphere = wet_atm_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; @@ -783,65 +783,87 @@ void MAMAci::set_grids( add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height + // BALLI:??? + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + grid_name); // cloud fraction [nondimentional] + + // ------------------------------------------------------------------------ // Output from this process + // ------------------------------------------------------------------------ + // BALLI:??? add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + // BALLI:??? add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + // BALLI:??? add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints // MUST FIXME: This should be an internal variable. why we need this as an // input??? + // BALLI:??? add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity - // MUST FIXME: move it above so that all "required" variables are together - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, - grid_name); // cloud fraction [nondimentional] + // BALLI:??? add_field("icenuc_num_hetfrz", scalar3d_layout_mid, 1 / m / m / m, grid_name); // number conc of ice nuclei due to - // heterogeneous freezing [1/m3] + // BALLI:??? // heterogeneous freezing [1/m3] add_field("icenuc_num_immfrz", scalar3d_layout_mid, 1 / m / m / m, grid_name); // number conc of ice nuclei due to immersion - // freezing (hetero nuc) [1/m3] + // BALLI:??? // freezing (hetero nuc) [1/m3] add_field("icenuc_num_depnuc", scalar3d_layout_mid, 1 / m / m / m, grid_name); // number conc of ice nuclei due to // deposition nucleation (hetero nuc)[1/m3] + // BALLI:??? add_field( "icenuc_num_meydep", scalar3d_layout_mid, 1 / m / m / m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + + // BALLI:??? add_field( "num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, n_unit, grid_name); // number of activated aerosol for ice nucleation // (homogeneous freezing only) [#/kg] + // BALLI:??? add_field( "num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] + // BALLI:??? add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] + // BALLI:??? add_field( "ptend_q", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne // aerosols [#/kg] + // BALLI:??? add_field( "tendnd", scalar3d_layout_mid, n_unit / s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] + // BALLI:??? add_field( "factnum", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam_coupling::num_aero_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] + // BALLI:??? add_field("ndropcol", scalar3d_layout_mid, n_unit / s, grid_name); // + // BALLI:??? add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); // droplet number mixing ratio tendency due // to mixing [#/kg/s] + // BALLI:??? add_field( "nsource", scalar3d_layout_mid, n_unit / s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] + // BALLI:??? add_field("wtke", scalar3d_layout_mid, n_unit / s, grid_name); // + + // BALLI:??? add_field( "ccn", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, @@ -851,10 +873,12 @@ void MAMAci::set_grids( // [#/m^2/s]] // / [aero. number conc. in updraft, just below // cloudbase [#/m^3]] + // BALLI:??? add_field( "coltend", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, nondim, grid_name); // column tendency for diagnostic output + // BALLI:??? add_field( "coltend_cw", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, @@ -871,7 +895,6 @@ void MAMAci::set_grids( mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - std::cout << "mode:" << mode << " name:" << int_nmr_field_name << std::endl; // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -880,8 +903,6 @@ void MAMAci::set_grids( mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - // std::cout<<"mode:"<< <<" name:"<< < 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - std::cout << "--> mode:" << mode << " spec:" << a - << " int-spec-name:" << int_mmr_field_name << std::endl; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -901,8 +920,6 @@ void MAMAci::set_grids( if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); - // std::cout<<"--> mode:"<< mode<<" spec:"<< a<< " cld-spec name:"<< - // cld_mmr_field_name<(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - // std::cout << "--> Gasses:" << g <<" spec:"<( "w_sec", scalar3d_layout_int, m2 / s2, grid_name); // Vertical velocity variance (wp2) at midpoints - + // BALLI:??? add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints + // BALLI:??? add_field( "liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints + // BALLI:??? add_field("kvh", scalar3d_layout_int, m2 / s, grid_name); // Eddy diffusivity for heat @@ -939,6 +957,7 @@ void MAMAci::set_grids( const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + // BALLI:??? add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols @@ -946,11 +965,13 @@ void MAMAci::set_grids( auto frz_unit = 1 / (cm * cm * cm * s); // units of number mixing ratios of tracers n_unit.set_string("1(cm^-3 s^-1)"); - + // BALLI:??? add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + // BALLI:??? add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + // BALLI:??? add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols // @@ -962,7 +983,7 @@ void MAMAci::set_grids( * qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). * qv_mid can be obtained from "get_field_in" call */ -} +} // set_grids ends void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( @@ -977,10 +998,6 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { } void MAMAci::initialize_impl(const RunType run_type) { - m_atm_logger->log(ekat::logger::LogLevel::info, "Calling ACI init"); - - // set atmosphere state data - w_sec_ = get_field_in("w_sec").get_view(); dgnum_ = get_field_in("dgnum") @@ -1010,34 +1027,34 @@ void MAMAci::initialize_impl(const RunType run_type) { coltend_outp_ = get_field_out("coltend").get_view(); coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - wet_atmosphere_.qv = get_field_in("qv").get_view(); - wet_atmosphere_.qc = get_field_in("qc").get_view(); - wet_atmosphere_.nc = get_field_in("nc").get_view(); - wet_atmosphere_.qi = get_field_in("qi").get_view(); - wet_atmosphere_.ni = get_field_in("ni").get_view(); - wet_atmosphere_.omega = get_field_in("omega").get_view(); - - dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); - dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); - dry_atmosphere_.p_int = get_field_in("p_int").get_view(); - dry_atmosphere_.p_del = - get_field_in("pseudo_density").get_view(); - dry_atmosphere_.qv = buffer_.qv_dry; - dry_atmosphere_.qc = buffer_.qc_dry; - dry_atmosphere_.nc = buffer_.nc_dry; - dry_atmosphere_.qi = buffer_.qi_dry; - dry_atmosphere_.ni = buffer_.ni_dry; - dry_atmosphere_.pblh = get_field_in("pbl_height").get_view(); - - dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) - dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above - // surface at interface levels (m) - dry_atmosphere_.z_mid = + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // pbl_height from SHOC + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + + dry_atm_.dz = buffer_.dz; // geometric thickness of layers (m) + dry_atm_.z_iface = buffer_.z_iface; // geopotential height above + // surface at interface levels (m) + dry_atm_.z_mid = buffer_.z_mid; // geopotential height above surface at mid levels (m) - dry_atmosphere_.cldfrac = - get_field_in("cldfrac_tot").get_view(); - dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.w_updraft = get_field_out("w_updraft").get_view(); hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); @@ -1137,8 +1154,8 @@ void MAMAci::initialize_impl(const RunType run_type) { hetfrz_.init(aero_config, hetfrz_config); // set up our preprocess functor - preprocess_.initialize(ncol_, nlev_, wet_atmosphere_, wet_aero_, - dry_atmosphere_, dry_aero_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } void MAMAci::run_impl(const double dt) { @@ -1157,8 +1174,8 @@ void MAMAci::run_impl(const double dt) { // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion - compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, - wet_atmosphere_, dry_atmosphere_, top_lev_, nlev_); + compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, wet_atm_, + dry_atm_, top_lev_, nlev_); compute_tke_using_w_sec(team_policy, tke_ /*output*/, w_sec_, nlev_); @@ -1176,10 +1193,10 @@ void MAMAci::run_impl(const double dt) { // FIXME: Find out in-outs of the following call! compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, dry_aero_, - dry_atmosphere_, aitken_dry_dia_, nlev_); + dry_atm_, aitken_dry_dia_, nlev_); store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, - wet_atmosphere_, liqcldf_, top_lev_); + wet_atm_, liqcldf_, top_lev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- @@ -1198,17 +1215,15 @@ void MAMAci::run_impl(const double dt) { rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, - nlev_); + compute_recipical_pseudo_density(team_policy, rpdel_, dry_atm_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. call_function_dropmixnuc( - team_policy, dry_atmosphere_, dry_aero_, dt, raercol_cw_, raercol_, - qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, - dry_atmosphere_.p_del, rpdel_, state_q_work_, dry_atmosphere_.nc, kvh_, - qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, - ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, - dropmixnuc_scratch_mem_, nlev_); + team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, raercol_, + qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atm_.p_int, + dry_atm_.p_del, rpdel_, state_q_work_, dry_atm_.nc, kvh_, qcld_, wsub_, + cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( @@ -1219,7 +1234,7 @@ void MAMAci::run_impl(const double dt) { team_policy, coltend_cw_outp_, coltend_cw_, nlev_); call_hetfrz_compute_tendencies( - team_policy, hetfrz_, dry_aero_, wet_atmosphere_, dry_atmosphere_, + team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, stratiform_cloud_fraction_, activation_fraction_accum_idx_, activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 3d30d797a82..931068cda82 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -46,10 +46,10 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d aitken_dry_dia_; // wet mixing ratios (water species) - mam_coupling::WetAtmosphere wet_atmosphere_; + mam_coupling::WetAtmosphere wet_atm_; // dry mixing ratios (water species) - mam_coupling::DryAtmosphere dry_atmosphere_; + mam_coupling::DryAtmosphere dry_atm_; // aerosol dry diameter const_view_3d dgnum_; From 4952a927e7bf8d8ece3858c64ee9a5221ec0f30d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 28 Mar 2024 16:58:37 -0700 Subject: [PATCH 050/142] Models compiles fine after the rebase off of optics branch --- .../mam/eamxx_mam_aci_process_interface.cpp | 24 ++-- .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 44 +----- .../tests/uncoupled/mam4/aci/CMakeLists.txt | 24 ---- .../eamxx/tests/uncoupled/mam4/aci/input.yaml | 131 ------------------ .../tests/uncoupled/mam4/aci/output.yaml | 12 -- 6 files changed, 20 insertions(+), 217 deletions(-) delete mode 100644 components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt delete mode 100644 components/eamxx/tests/uncoupled/mam4/aci/input.yaml delete mode 100644 components/eamxx/tests/uncoupled/mam4/aci/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index da7bd10e5c7..7b3def4ff24 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -374,7 +374,7 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], - MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, @@ -411,7 +411,7 @@ void call_function_dropmixnuc( MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; MAMAci::view_2d loc_qqcw[mam4::ndrop::ncnst_tot]; - MAMAci::view_2d loc_ptend_q[mam4::ndrop::nvar_ptend_q]; + MAMAci::view_2d loc_ptend_q[mam4::aero_model::pcnst]; MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; @@ -419,7 +419,7 @@ void call_function_dropmixnuc( for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; for(int i = 0; i < mam4::ndrop::pver; ++i) for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) + for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) @@ -470,8 +470,8 @@ void call_function_dropmixnuc( for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); } - mam4::ColumnView ptend_q_view[mam4::ndrop::nvar_ptend_q]; - for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { + mam4::ColumnView ptend_q_view[mam4::aero_model::pcnst]; + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); } mam4::ColumnView coltend_view[mam4::ndrop::ncnst_tot], @@ -501,14 +501,14 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::ndrop::nvars] = {}; // use pcnst here - Real qqcw_at_lev_col[mam4::ndrop::nvars] = {}; // use pcnst here + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here + Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); - for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { + for(int icnst = 15; icnst < mam4::aero_model::pcnst; ++icnst) { state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; // FIXME: ensure that indices are // right! remove "15" if possible!! @@ -836,7 +836,7 @@ void MAMAci::set_grids( // BALLI:??? add_field( "ptend_q", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::aero_model::pcnst}}, n_unit, grid_name); // tendencies for interstitial and cloud borne // aerosols [#/kg] // BALLI:??? @@ -956,7 +956,7 @@ void MAMAci::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ - {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points // BALLI:??? add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols @@ -1126,7 +1126,7 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(coltend_[i], ncol_, nlev_); Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } - for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { Kokkos::resize(ptend_q_[i], ncol_, nlev_); // MUST FIXME:Do we need this? } for(int i = 0; i < mam4::ndrop::pver; ++i) { @@ -1226,7 +1226,7 @@ void MAMAci::run_impl(const double dt) { nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. - copy_mam4xx_array_to_scream( + copy_mam4xx_array_to_scream( team_policy, ptend_q_output_, ptend_q_, nlev_); copy_mam4xx_array_to_scream( team_policy, coltend_outp_, coltend_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 931068cda82..0a35267ae3f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -70,7 +70,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d qcld_; view_2d tendnd_; // ptend_q_ is just ptend_q_output_ reformatted. - view_2d ptend_q_[mam4::ndrop::nvar_ptend_q]; + view_2d ptend_q_[mam4::aero_model::pcnst]; view_3d ptend_q_output_; view_3d factnum_; const_view_3d qqcw_input_; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 6463ff73e8b..4b71eeaf0f2 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -23,6 +23,9 @@ using const_view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; using const_view_3d = typename KT::template view_3d; +using complex_view_3d = typename KT::template view_3d>; +using complex_view_2d = typename KT::template view_2d>; + // Kokkos thread team (league member) using Team = Kokkos::TeamPolicy::member_type; @@ -266,10 +269,6 @@ struct WetAtmosphere { // This type stores multi-column views related to the dry atmospheric state // used by MAM. struct DryAtmosphere { -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index Real z_surf; // height of bottom of atmosphere [m] const_view_2d T_mid; // temperature at grid midpoints [K] const_view_2d p_mid; // total pressure at grid midpoints [Pa] @@ -282,40 +281,11 @@ struct DryAtmosphere { view_2d z_iface; // height at layer interfaces [m] view_2d dz; // layer thickness [m] const_view_2d p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] -<<<<<<< HEAD - const_view_2d p_int; // total pressure at grid interfaces [Pa] -======= const_view_2d p_int; // total pressure at grid interfaces [Pa] ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const_view_2d cldfrac; // cloud fraction [-] view_2d w_updraft; // updraft velocity [m/s] const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] -<<<<<<< HEAD -======= - Real z_surf; // height of bottom of atmosphere [m] - const_view_2d T_mid; // temperature at grid midpoints [K] - const_view_2d p_mid; // total pressure at grid midpoints [Pa] - view_2d qv; // dry water vapor mixing ratio [kg vapor / kg dry air] - view_2d qc; // dry cloud liquid water mass mixing ratio [kg cloud water/kg - // dry air] - view_2d nc; // dry cloud liquid water number mixing ratio [# / kg dry air] - view_2d qi; // dry cloud ice water mass mixing ratio [kg cloud ice water / kg - // dry air] - view_2d ni; // dry cloud ice water number mixing ratio [# / kg dry air] - view_2d z_mid; // height at layer midpoints [m] - view_2d z_iface; // height at layer interfaces [m] - view_2d dz; // layer thickness [m] - const_view_2d - p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] - const_view_2d p_int; // total pressure at grid interfaces [Pa] - const_view_2d cldfrac; // cloud fraction [-] - view_2d w_updraft; // updraft velocity [m/s] - const_view_1d pblh; // planetary boundary layer height [m] - const_view_1d phis; // surface geopotential [m2/s2] ->>>>>>> Modified species order to match E3SM with the folling change in mam4xx -======= ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index }; // This type stores aerosol number and mass mixing ratios evolved by MAM. It @@ -581,8 +551,8 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - int h_ind = mam4::utils::get_haero_prognostics_index(m,a);//Index of HAERO's prognostics - progs.q_aero_i[m][h_ind] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + int prg_idx = mam4::utils::get_prognostics_index(m,a);//Index of HAERO's prognostics + progs.q_aero_i[m][prg_idx] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); } } } @@ -607,8 +577,8 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.cld_aero_mmr[m][a].data()) { - int h_ind = mam4::utils::get_haero_prognostics_index(m,a); //Index of HAERO's prognostics array - progs.q_aero_c[m][h_ind] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + int prg_idx = mam4::utils::get_prognostics_index(m,a); //Index of HAERO's prognostics array + progs.q_aero_c[m][prg_idx] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); } } } diff --git a/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt deleted file mode 100644 index 45532850822..00000000000 --- a/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -include (ScreamUtils) - - - -# Create the test -CreateADUnitTest(mam4_aci_standalone - LABELS mam4 aci physics driver - LIBS mam - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} -) - -# Set AD configurable options -set (ATM_TIME_STEP 1) -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -set (RUN_T0 2021-10-12-45000) - -## Copy (and configure) yaml files needed by tests -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) - -# Ensure test input files are present in the data dir -GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) diff --git a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml deleted file mode 100644 index 298da6d50ad..00000000000 --- a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml +++ /dev/null @@ -1,131 +0,0 @@ -%YAML 1.1 ---- -driver_options: - atmosphere_dag_verbosity_level: 5 - -time_stepping: - time_step: ${ATM_TIME_STEP} - run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - #number_of_steps: ${NUM_STEPS} - number_of_steps: 1 - -atmosphere_processes: - atm_procs_list: [mam4_aci] - mam4_micro: - compute_tendencies: [] - -grids_manager: - Type: Mesh Free - grids_names: [Physics] - Physics: - type: point_grid - number_of_global_columns: 218 - number_of_vertical_levels: 72 - -initial_conditions: - # The name of the file containing the initial conditions for this test. FIXME: use a variable for the filename here - Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc - T_mid: 273.0 - p_mid: 1.e5 - p_int: 1.e5 - num_a1: 1.e5 - num_c1: 1.e5 - bc_a1: 1.e-5 - qqcw: 1e5 - pbl_height: 0.0 - - aero_gas_mmr_o3: 0.0 - aero_gas_mmr_h2o2: 0.0 - aero_gas_mmr_h2so4: 0.0 - aero_gas_mmr_so2: 0.0 - aero_gas_mmr_dms: 0.0 - aero_gas_mmr_soag: 0.0 - - hetfrz_immersion_nucleation_tend: 0 - hetfrz_contact_nucleation_tend: 0 - hetfrz_depostion_nucleation_tend: 0 - - zm: 1e1 - - w_updraft: 1e0 - ptend: 1e0 - - num_a2: 1e5 - num_a3: 1e5 - num_a4: 1e5 - num_c2: 1e5 - num_c3: 1e5 - num_c4: 1e5 - soa_a1: 1e-5 - so4_a1: 1e-5 - pom_a1: 1e-5 - nacl_a1: 1e-5 - dst_a1: 1e-5 - mom_a1: 1e-5 - soa_a2: 1e-5 - so4_a2: 1e-5 - nacl_a2: 1e-5 - mom_a2: 1e-5 - soa_a3: 1e-5 - so4_a3: 1e-5 - pom_a3: 1e-5 - bc_a3: 1e-5 - nacl_a3: 1e-5 - dst_a3: 1e-5 - mom_a3: 1e-5 - pom_a4: 1e-5 - bc_a4: 1e-5 - mom_a4: 1e-5 - #cloud borne aerosols - qi_coarse_dst: 1e-5 - qi_coarse_nacl: 1e-5 - qi_coarse_so4: 1e-5 - qi_coarse_mom: 1e-5 - qi_coarse_bc: 1e-5 - qi_coarse_pom: 1e-5 - qi_coarse_soa: 1e-5 - ni_accum: 1e-5 - ni_coarse: 1e-5 - ni_aitken: 1e-5 - - state_q: 1e-5 - ncldwtr: 1e-12 - - icenuc_num_hetfrz: 1e3 - icenuc_num_immfrz: 1e3 - icenuc_num_depnuc: 1e3 - icenuc_num_meydep: 1e3 - num_act_aerosol_ice_nucle_hom: 1e3 - num_act_aerosol_ice_nucle: 1e3 - - soa_c1: 1e-5 - so4_c1: 1e-5 - pom_c1: 1e-5 - bc_c1: 1e-5 - nacl_c1: 1e-5 - dst_c1: 1e-5 - mom_c1: 1e-5 - soa_c2: 1e-5 - so4_c2: 1e-5 - nacl_c2: 1e-5 - mom_c2: 1e-5 - soa_c3: 1e-5 - so4_c3: 1e-5 - pom_c3: 1e-5 - bc_c3: 1e-5 - nacl_c3: 1e-5 - dst_c3: 1e-5 - mom_c3: 1e-5 - pom_c4: 1e-5 - bc_c4: 1e-5 - mom_c4: 1e-5 - qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae - w_sec: 1e-2 - strat_cld_frac: 0.1 - liq_strat_cld_frac: 0.1 - kvh: 0.1 - dgnum: 1e-3 -# The parameters for I/O control -Scorpio: - output_yaml_files: ["output.yaml"] -... diff --git a/components/eamxx/tests/uncoupled/mam4/aci/output.yaml b/components/eamxx/tests/uncoupled/mam4/aci/output.yaml deleted file mode 100644 index 0087223d1d6..00000000000 --- a/components/eamxx/tests/uncoupled/mam4/aci/output.yaml +++ /dev/null @@ -1,12 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: mam4_aci_standalone_output -Averaging Type: Instant -Field Names: - - T_mid - - p_mid -output_control: - Frequency: 1 - frequency_units: nsteps - MPI Ranks in Filename: true -... From 59bf802d7b11a13c77224181e5a8f2e7227964fe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 30 Mar 2024 12:04:42 -0700 Subject: [PATCH 051/142] Single process test is working with a new init file now --- .../mam/eamxx_mam_aci_process_interface.cpp | 5 ++- .../eamxx/src/physics/mam/mam_coupling.hpp | 28 +++++------- components/eamxx/tests/CMakeLists.txt | 2 +- .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../single-process/mam/aci/CMakeLists.txt | 45 +++++++++++++++++++ .../tests/single-process/mam/aci/input.yaml | 43 ++++++++++++++++++ .../tests/single-process/mam/aci/output.yaml | 14 ++++++ 7 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 components/eamxx/tests/single-process/mam/aci/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/aci/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/aci/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7b3def4ff24..cddc51e6de8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -609,7 +609,6 @@ void call_hetfrz_compute_tendencies( Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // for(int icol = 0; icol < 1; icol++) { // Set up an atmosphere, surface, diagnostics, pronostics and // tendencies class. Real pblh = 0; @@ -705,7 +704,7 @@ void call_hetfrz_compute_tendencies( hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); }); - //} + //} } } // namespace @@ -1097,10 +1096,12 @@ void MAMAci::initialize_impl(const RunType run_type) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + std::cout<") +// mixing ratio field in EAMxx KOKKOS_INLINE_FUNCTION const char* gas_mmr_field_name(const int gas) { - if (!gas_mmr_names(gas)[0]) { - concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); - } - return const_cast(gas_mmr_names(gas)); + return const_cast(gas_species_name(gas)); } // This type stores multi-column views related specifically to the wet @@ -597,12 +594,12 @@ void compute_vertical_layer_heights(const Team& team, const auto dz = ekat::subview(dry_atm.dz, column_index); const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine - const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); - const auto qv = ekat::subview(dry_atm.qv, column_index); - + const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + // NOTE: we are using dry qv. Does calculate_dz require dry or wet? PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); @@ -612,7 +609,6 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } - // Given a thread team and wet and dry atmospheres, dispatches threads from the // team to compute the vertical updraft velocity for the column with the given // index. diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 7a7a9e82149..287fdf25af1 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -63,7 +63,7 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") - set(EAMxx_tests_IC_FILE_MAM4xx_72lev "scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc") + set(EAMxx_tests_IC_FILE_MAM4xx_72lev "screami_unit_tests_mam4xx_ne2np4L72_20240329.nc") # Testing individual atm processes add_subdirectory(single-process) diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index d417a9920f4..c6b4e0748af 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -20,6 +20,7 @@ if (SCREAM_ENABLE_MAM) # corresponding test here needs to be reworked with valid aerosol # initial conditions. add_subdirectory(mam/optics) + add_subdirectory(mam/aci) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt new file mode 100644 index 00000000000..2b33c95a517 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -0,0 +1,45 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_aci_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_aci physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_aci physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() \ No newline at end of file diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml new file mode 100644 index 00000000000..1e4c7327d71 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -0,0 +1,43 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_aci] + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + dgnum: 1e-3 + kvh: 1e-3 + liq_strat_cld_frac: 1e-3 + strat_cld_frac: 1e-3 + w_sec: 1e-3 + w_updraft: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/aci/output.yaml b/components/eamxx/tests/single-process/mam/aci/output.yaml new file mode 100644 index 00000000000..cb54b742c5a --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aci/output.yaml @@ -0,0 +1,14 @@ +%YAML 1.1 +--- +filename_prefix: mam4_aci_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - T_mid + +output_control: + Frequency: 2 + frequency_units: nsteps + MPI Ranks in Filename: true +... From 1afbf982e9e52d0e7c487faaff3497d91f968727 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 1 Apr 2024 17:29:20 -0700 Subject: [PATCH 052/142] Removes some variables from the input file and cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 111 ++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- .../tests/single-process/mam/aci/input.yaml | 1 - 3 files changed, 50 insertions(+), 66 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index cddc51e6de8..31aaf9025a9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -419,8 +419,7 @@ void call_function_dropmixnuc( for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; for(int i = 0; i < mam4::ndrop::pver; ++i) for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - for(int i = 0; i < mam4::aero_model::pcnst; ++i) - loc_ptend_q[i] = ptend_q[i]; + for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend_cw[i] = coltend_cw[i]; @@ -501,8 +500,9 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here - Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here + Real state_q_at_lev_col[mam4::aero_model::pcnst] = + {}; // use pcnst here + Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); @@ -583,9 +583,7 @@ void call_hetfrz_compute_tendencies( mam_coupling::AerosolState &dry_aero_, mam_coupling::WetAtmosphere &wet_atm_, mam_coupling::DryAtmosphere &dry_atm_, - MAMAci::view_2d stratiform_cloud_fraction, - MAMAci::view_2d activation_fraction_accum_idx, - MAMAci::view_2d activation_fraction_coarse_idx, + MAMAci::view_3d factnum_, MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, @@ -594,6 +592,7 @@ void call_hetfrz_compute_tendencies( using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); + mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; mam_coupling::WetAtmosphere wet_atmosphere = wet_atm_; @@ -601,21 +600,21 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; - MAMAci::const_view_2d qc = wet_atmosphere.qc; - MAMAci::const_view_2d nc = wet_atmosphere.nc; - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); + //for (int icol=0; icol<1; icol++){ // Set up an atmosphere, surface, diagnostics, pronostics and // tendencies class. - Real pblh = 0; + + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + /*Real pblh = 0; haero::Atmosphere atmos( nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, - dummy, dummy, dummy, dummy, dummy, pblh); + dummy, dummy, dummy, dummy, dummy, pblh);*/ // set surface state data haero::Surface surf{}; mam4::Prognostics progs = @@ -626,12 +625,10 @@ void call_hetfrz_compute_tendencies( // BALLI mam4::Diagnostics diags(nlev); - diags.stratiform_cloud_fraction = - ekat::subview(stratiform_cloud_fraction, icol); - diags.activation_fraction[accum_idx] = - ekat::subview(activation_fraction_accum_idx, icol); - diags.activation_fraction[coarse_idx] = - ekat::subview(activation_fraction_coarse_idx, icol); + + diags.activation_fraction[accum_idx] = ekat::subview(factnum_,icol, accum_idx); + + diags.activation_fraction[coarse_idx] = ekat::subview(factnum_, icol, coarse_idx); // These are the output tendencies from heterogeneous freezing that need // to be added correctly to the cloud-micorphysics scheme. @@ -701,10 +698,10 @@ void call_hetfrz_compute_tendencies( const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t = 0; //, dt = 0; - hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, + hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, diags, tends); }); - //} + //} } } // namespace @@ -723,8 +720,9 @@ void MAMAci::set_grids( ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + // Allocate memory for the class members // Kokkos::resize only works on host to allocates memory - Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(rho_, ncol_, nlev_);// FIXME: rho_ is only used internally in compute_w0_and_rho Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -766,6 +764,7 @@ void MAMAci::set_grids( "tracers"); // cloud liquid number mixing ratio [1/kg] add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // cloud ice number mixing ratio [1/kg] + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints @@ -782,28 +781,13 @@ void MAMAci::set_grids( add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height - // BALLI:??? + // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, - grid_name); // cloud fraction [nondimentional] + grid_name); // ------------------------------------------------------------------------ // Output from this process // ------------------------------------------------------------------------ - // BALLI:??? - add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, - grid_name); // Layer thickness(pdel) [Pa] at midpoints - // BALLI:??? - add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, - grid_name); // Layer thickness(pdel) [Pa] at midpoints - // BALLI:??? - add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, - grid_name); // Layer thickness(pdel) [Pa] at midpoints - - // MUST FIXME: This should be an internal variable. why we need this as an - // input??? - // BALLI:??? - add_field("w_updraft", scalar3d_layout_mid, q_unit, - grid_name); // updraft velocity // BALLI:??? add_field("icenuc_num_hetfrz", scalar3d_layout_mid, 1 / m / m / m, @@ -845,8 +829,8 @@ void MAMAci::set_grids( // BALLI:??? add_field( "factnum", - FieldLayout{{COL, LEV, CMP}, - {ncol_, nlev_, mam_coupling::num_aero_modes()}}, + FieldLayout{{COL, NMODES, LEV}, + {ncol_, mam_coupling::num_aero_modes(),nlev_}}, nondim, grid_name); // activation fraction for aerosol number [fraction] // BALLI:??? add_field("ndropcol", scalar3d_layout_mid, n_unit / s, @@ -941,9 +925,7 @@ void MAMAci::set_grids( add_field( "w_sec", scalar3d_layout_int, m2 / s2, grid_name); // Vertical velocity variance (wp2) at midpoints - // BALLI:??? - add_field("strat_cld_frac", scalar3d_layout_mid, nondim, - grid_name); // Stratiform cloud fraction at midpoints + // BALLI:??? add_field( "liq_strat_cld_frac", scalar3d_layout_mid, nondim, @@ -961,18 +943,22 @@ void MAMAci::set_grids( grid_name); // dry diameter of aerosols auto cm = m / 100; - auto frz_unit = - 1 / (cm * cm * cm * s); // units of number mixing ratios of tracers + + // units of number mixing ratios of tracers + auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); - // BALLI:??? + + // heterogeous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, - frz_unit, grid_name); // dry diameter of aerosols - // BALLI:??? + frz_unit, grid_name); + + // heterogeous freezing by contact nucleation [cm^-3 s^-1] add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, - frz_unit, grid_name); // dry diameter of aerosols - // BALLI:??? + frz_unit, grid_name); + + // heterogeous freezing by deposition nucleation [cm^-3 s^-1] add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, - frz_unit, grid_name); // dry diameter of aerosols + frz_unit, grid_name); // /* * NOTE on other inputs for the aci process: @@ -1053,7 +1039,7 @@ void MAMAci::initialize_impl(const RunType run_type) { buffer_.z_mid; // geopotential height above surface at mid levels (m) dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); - dry_atm_.w_updraft = get_field_out("w_updraft").get_view(); + dry_atm_.w_updraft = buffer_.w_updraft; hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); @@ -1062,13 +1048,6 @@ void MAMAci::initialize_impl(const RunType run_type) { hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); - stratiform_cloud_fraction_ = - get_field_out("stratiform_cloud_fraction").get_view(); - activation_fraction_accum_idx_ = - get_field_out("activation_fraction_accum").get_view(); - activation_fraction_coarse_idx_ = - get_field_out("activation_fraction_coarse").get_view(); - // allocate work // FIXME: store 25 into a const var or get from MAM4xx for(int icnst = 0; icnst < 25; ++icnst) { @@ -1096,12 +1075,14 @@ void MAMAci::initialize_impl(const RunType run_type) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - + if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; - std::cout<( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1236,8 +1220,7 @@ void MAMAci::run_impl(const double dt) { call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, - stratiform_cloud_fraction_, activation_fraction_accum_idx_, - activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, + factnum_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, naai_hom_, naai_, diagnostic_scratch_, nlev_, dt); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 0a35267ae3f..be7d116fe5e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -25,8 +25,10 @@ class MAMAci final : public scream::AtmosphereProcess { mam4::Hetfrz hetfrz_; // views for single- and multi-column data + using view_1d = scream::mam_coupling::view_1d; using view_2d = scream::mam_coupling::view_2d; using view_3d = scream::mam_coupling::view_3d; + using const_view_1d = scream::mam_coupling::const_view_1d; using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; @@ -196,7 +198,7 @@ class MAMAci final : public scream::AtmosphereProcess { // vertical heights has to be computed after computing dry mixing ratios // for atmosphere compute_vertical_layer_heights(team, dry_atm_pre_, i); - + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); } // operator() // local variables for preprocess struct diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index 1e4c7327d71..b63060e04a5 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -33,7 +33,6 @@ initial_conditions: dgnum: 1e-3 kvh: 1e-3 liq_strat_cld_frac: 1e-3 - strat_cld_frac: 1e-3 w_sec: 1e-3 w_updraft: 1e-3 From 94cd1fa7b85687ea1b350c6ec657703f7cd1c621 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 1 Apr 2024 20:19:52 -0700 Subject: [PATCH 053/142] Cleanup of some I/O variables for ice nuc and dropmixnuc processes --- .../mam/eamxx_mam_aci_process_interface.cpp | 142 +++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- 2 files changed, 76 insertions(+), 70 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 31aaf9025a9..25afa6012e4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -386,7 +386,7 @@ void call_function_dropmixnuc( MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { - // FIXME: why can't we use MAMAci::drop_scratch_ above + // FIXME: why can't we use MAMAci::dropmix_scratch_ above MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -582,8 +582,7 @@ void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, mam_coupling::AerosolState &dry_aero_, mam_coupling::WetAtmosphere &wet_atm_, - mam_coupling::DryAtmosphere &dry_atm_, - MAMAci::view_3d factnum_, + mam_coupling::DryAtmosphere &dry_atm_, MAMAci::view_3d factnum_, MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, @@ -592,7 +591,6 @@ void call_hetfrz_compute_tendencies( using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); - mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; mam_coupling::WetAtmosphere wet_atmosphere = wet_atm_; @@ -600,13 +598,12 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; - Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - //for (int icol=0; icol<1; icol++){ - // Set up an atmosphere, surface, diagnostics, pronostics and - // tendencies class. + // for (int icol=0; icol<1; icol++){ + // Set up an atmosphere, surface, diagnostics, pronostics and + // tendencies class. haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); @@ -626,9 +623,11 @@ void call_hetfrz_compute_tendencies( // BALLI mam4::Diagnostics diags(nlev); - diags.activation_fraction[accum_idx] = ekat::subview(factnum_,icol, accum_idx); + diags.activation_fraction[accum_idx] = + ekat::subview(factnum_, icol, accum_idx); - diags.activation_fraction[coarse_idx] = ekat::subview(factnum_, icol, coarse_idx); + diags.activation_fraction[coarse_idx] = + ekat::subview(factnum_, icol, coarse_idx); // These are the output tendencies from heterogeneous freezing that need // to be added correctly to the cloud-micorphysics scheme. @@ -682,11 +681,6 @@ void call_hetfrz_compute_tendencies( diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - //------------------------------------------------------------- // Heterogeneous freezing // frzimm, frzcnt, frzdep are the outputs of @@ -698,8 +692,8 @@ void call_hetfrz_compute_tendencies( const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t = 0; //, dt = 0; - hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, - diags, tends); + hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, + progs, diags, tends); }); //} } @@ -722,7 +716,9 @@ void MAMAci::set_grids( // Allocate memory for the class members // Kokkos::resize only works on host to allocates memory - Kokkos::resize(rho_, ncol_, nlev_);// FIXME: rho_ is only used internally in compute_w0_and_rho + Kokkos::resize( + rho_, ncol_, + nlev_); // FIXME: rho_ is only used internally in compute_w0_and_rho Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -734,7 +730,7 @@ void MAMAci::set_grids( Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); Kokkos::resize(rpdel_, ncol_, nlev_); - for(int i = 0; i < drop_scratch_; ++i) { + for(int i = 0; i < dropmix_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } // Define the different field layouts that will be used for this process @@ -768,9 +764,9 @@ void MAMAci::set_grids( add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints add_field("p_int", scalar3d_layout_int, Pa, @@ -782,58 +778,71 @@ void MAMAci::set_grids( grid_name); // planetary boundary layer height // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, - grid_name); + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + + // ======================================================================== + // Output from this whole process + // ======================================================================== // ------------------------------------------------------------------------ - // Output from this process + // Output from ice nucleation process // ------------------------------------------------------------------------ + auto m3_inv = 1 / m / m / m; // inverse of m3 + // number conc of ice nuclei due to heterogeneous freezing [1/m3] + add_field("icenuc_num_hetfrz", scalar3d_layout_mid, m3_inv, + grid_name); - // BALLI:??? - add_field("icenuc_num_hetfrz", scalar3d_layout_mid, 1 / m / m / m, - grid_name); // number conc of ice nuclei due to - // BALLI:??? // heterogeneous freezing [1/m3] - add_field("icenuc_num_immfrz", scalar3d_layout_mid, 1 / m / m / m, - grid_name); // number conc of ice nuclei due to immersion - // BALLI:??? // freezing (hetero nuc) [1/m3] - add_field("icenuc_num_depnuc", scalar3d_layout_mid, 1 / m / m / m, - grid_name); // number conc of ice nuclei due to - // deposition nucleation (hetero nuc)[1/m3] - // BALLI:??? - add_field( - "icenuc_num_meydep", scalar3d_layout_mid, 1 / m / m / m, - grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + // number conc of ice nuclei due to immersionfreezing (hetero nuc) [1/m3] + add_field("icenuc_num_immfrz", scalar3d_layout_mid, m3_inv, + grid_name); + // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] + add_field("icenuc_num_depnuc", scalar3d_layout_mid, m3_inv, + grid_name); // + + // number conc of ice nuclei due to meyers deposition [1/m3] + add_field("icenuc_num_meydep", scalar3d_layout_mid, m3_inv, + grid_name); + + // number of activated aerosol for ice nucleation(homogeneous frz only)[#/kg] + add_field("num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, + n_unit, grid_name); + + // number of activated aerosol for ice nucleation[#/kg] + add_field("num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, + grid_name); + + // ------------------------------------------------------------------------ + // Output from droplet activation process (dropmixnuc) + // ------------------------------------------------------------------------ // BALLI:??? - add_field( - "num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, n_unit, - grid_name); // number of activated aerosol for ice nucleation - // (homogeneous freezing only) [#/kg] - // BALLI:??? - add_field( - "num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, - grid_name); // number of activated aerosol for ice nucleation[#/kg] - // BALLI:??? + // FIXME: THis looks like an internal variable for dropmixnuc, why we need it + // here??? add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] // BALLI:??? + // FIXME:This array should update the mmrs and nmrs + // tendencies for interstitial and cloud borne aerosols [#/kg] add_field( "ptend_q", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::aero_model::pcnst}}, - n_unit, grid_name); // tendencies for interstitial and cloud borne - // aerosols [#/kg] - // BALLI:??? - add_field( - "tendnd", scalar3d_layout_mid, n_unit / s, - grid_name); // tendency in droplet number mixing ratio [#/kg/s] - // BALLI:??? + n_unit, grid_name); + + // tendency in droplet number mixing ratio [#/kg/s] + add_field("tendnd", scalar3d_layout_mid, n_unit / s, grid_name); + + // activation fraction for aerosol number [fraction] add_field( "factnum", FieldLayout{{COL, NMODES, LEV}, - {ncol_, mam_coupling::num_aero_modes(),nlev_}}, - nondim, grid_name); // activation fraction for aerosol number [fraction] - // BALLI:??? - add_field("ndropcol", scalar3d_layout_mid, n_unit / s, + {ncol_, mam_coupling::num_aero_modes(), nlev_}}, + nondim, grid_name); + + auto inv_m2 = 1 / m / m; // units of number mixing ratios of tracers + inv_m2.set_string("#/m2"); + // BALLI:??? FIXME: This is internal diagnostic variable + // column-integrated droplet number [#/m2] + add_field("ndropcol", scalar3d_layout_mid, inv_m2, grid_name); // // BALLI:??? add_field("ndropmix", scalar3d_layout_mid, n_unit / s, @@ -944,10 +953,10 @@ void MAMAci::set_grids( auto cm = m / 100; - // units of number mixing ratios of tracers + // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); - + // heterogeous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); @@ -1207,9 +1216,6 @@ void MAMAci::run_impl(const double dt) { cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. - - - copy_mam4xx_array_to_scream( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1219,10 +1225,10 @@ void MAMAci::run_impl(const double dt) { team_policy, coltend_cw_outp_, coltend_cw_, nlev_); call_hetfrz_compute_tendencies( - team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, - factnum_, hetfrz_immersion_nucleation_tend_, - hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, - naai_hom_, naai_, diagnostic_scratch_, nlev_, dt); + team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, factnum_, + hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, + hetfrz_depostion_nucleation_tend_, naai_hom_, naai_, diagnostic_scratch_, + nlev_, dt); Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index be7d116fe5e..a73e04b8575 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -95,8 +95,8 @@ class MAMAci final : public scream::AtmosphereProcess { view_3d nact_; view_3d mact_; - static constexpr int drop_scratch_ = 15; - view_2d dropmixnuc_scratch_mem_[drop_scratch_]; + static constexpr int dropmix_scratch_ = 15; + view_2d dropmixnuc_scratch_mem_[dropmix_scratch_]; view_2d stratiform_cloud_fraction_; view_2d activation_fraction_accum_idx_; From 3c90273c9a8f13970ee1e37f8e86e7ffef0ca163 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 1 Apr 2024 22:18:52 -0700 Subject: [PATCH 054/142] Further cleanup and code lines moved to match I/O for the process --- .../mam/eamxx_mam_aci_process_interface.cpp | 182 +++++++++--------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 25afa6012e4..636aee9433f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -780,10 +780,74 @@ void MAMAci::set_grids( // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and + // BALLI:??? + // Vertical velocity variance (wp2) at midpoints + add_field("w_sec", scalar3d_layout_int, m2 / s2, grid_name); + + // BALLI:??? + // FIXME:liq_strat_cld_frac may also need OLD time + // Liquid stratiform cloud fraction at midpoints + add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, + grid_name); + // BALLI:??? + add_field("kvh", scalar3d_layout_int, m2 / s, + grid_name); // Eddy diffusivity for heat + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + const int num_aero_modes = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_layout_mid{ + {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + // BALLI:??? + add_field("dgnum", scalar4d_layout_mid, m, + grid_name); // dry diameter of aerosols + // ======================================================================== // Output from this whole process // ======================================================================== + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(mode, a); + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(mode, a); + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + // ------------------------------------------------------------------------ // Output from ice nucleation process // ------------------------------------------------------------------------ @@ -838,88 +902,47 @@ void MAMAci::set_grids( {ncol_, mam_coupling::num_aero_modes(), nlev_}}, nondim, grid_name); - auto inv_m2 = 1 / m / m; // units of number mixing ratios of tracers + auto inv_m2 = 1 / m / m; inv_m2.set_string("#/m2"); // BALLI:??? FIXME: This is internal diagnostic variable // column-integrated droplet number [#/m2] add_field("ndropcol", scalar3d_layout_mid, inv_m2, grid_name); // - // BALLI:??? - add_field("ndropmix", scalar3d_layout_mid, n_unit / s, - grid_name); // droplet number mixing ratio tendency due - // to mixing [#/kg/s] - // BALLI:??? - add_field( - "nsource", scalar3d_layout_mid, n_unit / s, - grid_name); // droplet number mixing ratio source tendency [#/kg/s] - // BALLI:??? - add_field("wtke", scalar3d_layout_mid, n_unit / s, grid_name); // + // BALLI:??? FIXME: This is internal diagnostic variable + // droplet number mixing ratio tendency due to mixing [#/kg/s] + add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); + // BALLI:??? FIXME: This is internal diagnostic variable + // droplet number mixing ratio source tendency [#/kg/s] + add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); - // BALLI:??? + // BALLI:??? FIXME: This is internal diagnostic variable + // subgrid vertical velocity [m/s] + add_field("wtke", scalar3d_layout_mid, m / s, grid_name); + + // BALLI:??? FIXME: This is internal diagnostic variable + // number conc of aerosols activated at supersat [#/m^3] + // note: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud + // [#/m^2/s]] + // / [aero. number conc. in updraft, just below + // cloudbase [#/m^3]] add_field( "ccn", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, - n_unit, - grid_name); // number conc of aerosols activated at supersat [#/m^3] - // note: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud - // [#/m^2/s]] - // / [aero. number conc. in updraft, just below - // cloudbase [#/m^3]] - // BALLI:??? + m3_inv, grid_name); + + // BALLI:??? FIXME: This is internal diagnostic variable + // column tendency for diagnostic output add_field( "coltend", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - nondim, grid_name); // column tendency for diagnostic output - // BALLI:??? + nondim, grid_name); + + // BALLI:??? FIXME: This is internal diagnostic variable + // column tendency add_field( "coltend_cw", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - nondim, grid_name); // column tendency - - // MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to - // dry - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios - for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { - // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = - mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); - - // cloudborne aerosol tracers of interest: number (n) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected - const char *cld_nmr_field_name = - mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); - - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(mode, a); - if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(mode, a); - if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); - } - } - } - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } + nondim, grid_name); // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM // These outputs should come from the cloud macrophysics process (e.g., SHOC) @@ -928,29 +951,6 @@ void MAMAci::set_grids( auto s2 = s * s; s2.set_string("s^2"); - // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and - // liq_strat_cld_frac may also need OLD time - // BALLI:??? - add_field( - "w_sec", scalar3d_layout_int, m2 / s2, - grid_name); // Vertical velocity variance (wp2) at midpoints - - // BALLI:??? - add_field( - "liq_strat_cld_frac", scalar3d_layout_mid, nondim, - grid_name); // Liquid stratiform cloud fraction at midpoints - // BALLI:??? - add_field("kvh", scalar3d_layout_int, m2 / s, - grid_name); // Eddy diffusivity for heat - - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables - const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ - {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points - // BALLI:??? - add_field("dgnum", scalar4d_layout_mid, m, - grid_name); // dry diameter of aerosols - auto cm = m / 100; // units of number mixing ratios of tracers From 08430d5249a0dfde88eac5f6f60250d960838f43 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 1 Apr 2024 22:54:43 -0700 Subject: [PATCH 055/142] Adds eddy diff of heat as an output for SHOC --- .../mam/eamxx_mam_aci_process_interface.cpp | 26 ++++++------ .../shoc/eamxx_shoc_process_interface.cpp | 41 +++++++++++-------- .../src/physics/shoc/impl/shoc_main_impl.hpp | 7 +++- .../eamxx/src/physics/shoc/shoc_functions.hpp | 2 + .../tests/single-process/mam/aci/input.yaml | 2 +- 5 files changed, 45 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 636aee9433f..840d3eb181a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -780,6 +780,12 @@ void MAMAci::set_grids( // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM + // These outputs should come from the cloud macrophysics process (e.g., SHOC) + auto m2 = m * m; + m2.set_string("m^2"); + auto s2 = s * s; + s2.set_string("s^2"); // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and // BALLI:??? // Vertical velocity variance (wp2) at midpoints @@ -791,7 +797,7 @@ void MAMAci::set_grids( add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // BALLI:??? - add_field("kvh", scalar3d_layout_int, m2 / s, + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables @@ -944,13 +950,6 @@ void MAMAci::set_grids( FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, nondim, grid_name); - // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM - // These outputs should come from the cloud macrophysics process (e.g., SHOC) - auto m2 = m * m; - m2.set_string("m^2"); - auto s2 = s * s; - s2.set_string("s^2"); - auto cm = m / 100; // units of number mixing ratios of tracers @@ -994,13 +993,12 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { void MAMAci::initialize_impl(const RunType run_type) { w_sec_ = get_field_in("w_sec").get_view(); - dgnum_ = get_field_in("dgnum") - .get_view(); // MUST FIXME: is it an input, can - // we compute it using calcsize??? + // MUST FIXME: is it an input, should we invoke calcsize here?? + dgnum_ = get_field_in("dgnum").get_view(); liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); - kvh_ = get_field_in("kvh") - .get_view(); // MUST FIXME: See if scream has it, - // it should com from the land model + + // MUST FIXME: This comes from shoc + kvh_ = get_field_in("eddy_diff_mom").get_view(); nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); niim_ = get_field_out("icenuc_num_immfrz").get_view(); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 6204421c1ea..4c44bc32fe7 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -23,6 +23,12 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids { using namespace ekat::units; + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto Qunit = kg/kg; + Qunit.set_string("kg/kg"); + auto nondim = Units::nondimensional(); + m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -50,22 +56,21 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids constexpr int ps = Spack::n; - const auto nondim = Units::nondimensional(); - const auto m2 = pow(m,2); - const auto s2 = pow(s,2); + const auto m2 = m*m; + const auto s2 = s*s; // These variables are needed by the interface, but not actually passed to shoc_main. add_field("omega", scalar3d_mid, Pa/s, grid_name, ps); add_field("surf_sens_flux", scalar2d , W/m2, grid_name); add_field("surf_mom_flux", vector2d , N/m2, grid_name); - add_field("surf_evap", scalar2d , kg/(m2*s), grid_name); - add_field ("T_mid", scalar3d_mid, K, grid_name, ps); - add_field ("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); + add_field("surf_evap", scalar2d , kg/m2/s, grid_name); + add_field ("T_mid", scalar3d_mid, K, grid_name, ps); + add_field ("qv", scalar3d_mid, Qunit, grid_name, "tracers", ps); // If TMS is a process, add surface drag coefficient to required fields if (m_params.get("apply_tms", false)) { - add_field("surf_drag_coeff_tms", scalar2d, kg/(m2*s), grid_name); + add_field("surf_drag_coeff_tms", scalar2d, kg/s/m2, grid_name); } // Input variables @@ -75,23 +80,24 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d , m, grid_name); - add_field("inv_qc_relvar", scalar3d_mid, pow(kg/kg,2), grid_name, ps); + add_field("pbl_height", scalar2d , m, grid_name); + add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { - add_field("vapor_flux", scalar2d, kg/(m2*s), grid_name); + add_field("vapor_flux", scalar2d, kg/m2/s, grid_name); add_field("water_flux", scalar2d, m/s, grid_name); add_field("ice_flux", scalar2d, m/s, grid_name); add_field("heat_flux", scalar2d, W/m2, grid_name); @@ -252,6 +258,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto& cldfrac_liq = get_field_out("cldfrac_liq").get_view(); const auto& sgs_buoy_flux = get_field_out("sgs_buoy_flux").get_view(); const auto& tk = get_field_out("eddy_diff_mom").get_view(); + const auto& tkh = get_field_out("eddy_diff_heat").get_view(); const auto& inv_qc_relvar = get_field_out("inv_qc_relvar").get_view(); const auto& phis = get_field_in("phis").get_view(); @@ -285,6 +292,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial){ Kokkos::deep_copy(sgs_buoy_flux,0.0); Kokkos::deep_copy(tk,0.0); + Kokkos::deep_copy(tkh,0.0); Kokkos::deep_copy(tke,0.0004); Kokkos::deep_copy(tke_copy,0.0004); Kokkos::deep_copy(cldfrac_liq,0.0); @@ -321,6 +329,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) input_output.wthv_sec = sgs_buoy_flux; input_output.qtracers = shoc_preprocess.qtracers; input_output.tk = tk; + input_output.tkh = tkh; input_output.shoc_cldfrac = cldfrac_liq; input_output.shoc_ql = qc_copy; diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 13a93992cff..bb19677eb27 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -114,6 +114,7 @@ void Functions::shoc_main_internal( const uview_1d& wthv_sec, const uview_2d& qtracers, const uview_1d& tk, + const uview_1d& tkh, const uview_1d& shoc_cldfrac, const uview_1d& shoc_ql, // Output Variables @@ -366,6 +367,7 @@ void Functions::shoc_main_internal( const view_2d& wthv_sec, const view_3d& qtracers, const view_2d& tk, + const view_2d& tkh, const view_2d& shoc_cldfrac, const view_2d& shoc_ql, // Output Variables @@ -640,6 +642,7 @@ Int Functions::shoc_main( const auto qw_s = ekat::subview(shoc_input_output.qw, i); const auto wthv_sec_s = ekat::subview(shoc_input_output.wthv_sec, i); const auto tk_s = ekat::subview(shoc_input_output.tk, i); + const auto tkh_s = ekat::subview(shoc_input_output.tkh, i); const auto shoc_cldfrac_s = ekat::subview(shoc_input_output.shoc_cldfrac, i); const auto shoc_ql_s = ekat::subview(shoc_input_output.shoc_ql, i); const auto shoc_ql2_s = ekat::subview(shoc_output.shoc_ql2, i); @@ -672,7 +675,7 @@ Int Functions::shoc_main( wtracer_sfc_s, inv_exner_s, phis_s, // Input workspace, // Workspace host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output - wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output + wthv_sec_s, qtracers_s, tk_s, tkh_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output pblh_s, shoc_ql2_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables @@ -696,7 +699,7 @@ Int Functions::shoc_main( shoc_input.wtracer_sfc, shoc_input.inv_exner, shoc_input.phis, // Input workspace_mgr, // Workspace Manager shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output - shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output + shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.tkh, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output shoc_output.pblh, shoc_output.shoc_ql2, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index d02d498c5a4..2d9744b9aed 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -147,6 +147,8 @@ struct Functions view_3d qtracers; // eddy coefficient for momentum [m2/s] view_2d tk; + // eddy coefficient for heat [m2/s] + view_2d tkh; // Cloud fraction [-] view_2d shoc_cldfrac; // cloud liquid mixing ratio [kg/kg] diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index b63060e04a5..f21bde0ce3f 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -31,7 +31,7 @@ initial_conditions: #we should get the following variables from other processes dgnum: 1e-3 - kvh: 1e-3 + eddy_diff_heat: 1e-3 liq_strat_cld_frac: 1e-3 w_sec: 1e-3 w_updraft: 1e-3 From bb0e696853682e92fa0d4fd6b5d6286f1e8bb921 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Apr 2024 12:01:53 -0700 Subject: [PATCH 056/142] Adds tkh-eddy diffusivity of heat in SHOC-2nd attempt --- .../shoc/eamxx_shoc_process_interface.cpp | 10 ++++---- .../src/physics/shoc/impl/shoc_main_impl.hpp | 24 ++++++++++++------- .../eamxx/src/physics/shoc/shoc_functions.hpp | 6 +++-- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 4c44bc32fe7..a988948c0f1 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -84,13 +84,13 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d , m, grid_name); - add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); + add_field("pbl_height", scalar2d_layout_col, m, grid_name); + add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -258,7 +258,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto& cldfrac_liq = get_field_out("cldfrac_liq").get_view(); const auto& sgs_buoy_flux = get_field_out("sgs_buoy_flux").get_view(); const auto& tk = get_field_out("eddy_diff_mom").get_view(); - const auto& tkh = get_field_out("eddy_diff_heat").get_view(); const auto& inv_qc_relvar = get_field_out("inv_qc_relvar").get_view(); const auto& phis = get_field_in("phis").get_view(); @@ -292,7 +291,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial){ Kokkos::deep_copy(sgs_buoy_flux,0.0); Kokkos::deep_copy(tk,0.0); - Kokkos::deep_copy(tkh,0.0); Kokkos::deep_copy(tke,0.0004); Kokkos::deep_copy(tke_copy,0.0004); Kokkos::deep_copy(cldfrac_liq,0.0); @@ -329,13 +327,13 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) input_output.wthv_sec = sgs_buoy_flux; input_output.qtracers = shoc_preprocess.qtracers; input_output.tk = tk; - input_output.tkh = tkh; input_output.shoc_cldfrac = cldfrac_liq; input_output.shoc_ql = qc_copy; // Output Variables output.pblh = get_field_out("pbl_height").get_view(); output.shoc_ql2 = shoc_ql2; + output.tkh = get_field_out("eddy_diff_heat").get_view(); // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index bb19677eb27..23348a19f50 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -114,12 +114,12 @@ void Functions::shoc_main_internal( const uview_1d& wthv_sec, const uview_2d& qtracers, const uview_1d& tk, - const uview_1d& tkh, const uview_1d& shoc_cldfrac, const uview_1d& shoc_ql, // Output Variables Scalar& pblh, const uview_1d& shoc_ql2, + const uview_1d& tkh_out, // Diagnostic Output Variables const uview_1d& shoc_mix, const uview_1d& w_sec, @@ -266,7 +266,7 @@ void Functions::shoc_main_internal( check_tke(team,nlev,tke); } - // End SHOC parameterization + // End SHOC parameterization // Use SHOC outputs to update the host model // temperature @@ -311,6 +311,12 @@ void Functions::shoc_main_internal( workspace, // Workspace pblh); // Output + // Assign tkh to tkh output variable + const Int nlev_pack = ekat::npack(nlev); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + tkh_out(k) = tkh(k); + }); + // Release temporary variables from the workspace workspace.template release_many_contiguous<6>( {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi, &tkh}); @@ -367,12 +373,12 @@ void Functions::shoc_main_internal( const view_2d& wthv_sec, const view_3d& qtracers, const view_2d& tk, - const view_2d& tkh, const view_2d& shoc_cldfrac, const view_2d& shoc_ql, // Output Variables const view_1d& pblh, const view_2d& shoc_ql2, + const view_2d& tkh_out, // Diagnostic Output Variables const view_2d& shoc_mix, const view_2d& w_sec, @@ -567,6 +573,8 @@ void Functions::shoc_main_internal( kbfs,shoc_cldfrac, // Input workspace_mgr, // Workspace mgr pblh); // Output + + tkh_out = tkh; } #endif @@ -642,10 +650,10 @@ Int Functions::shoc_main( const auto qw_s = ekat::subview(shoc_input_output.qw, i); const auto wthv_sec_s = ekat::subview(shoc_input_output.wthv_sec, i); const auto tk_s = ekat::subview(shoc_input_output.tk, i); - const auto tkh_s = ekat::subview(shoc_input_output.tkh, i); const auto shoc_cldfrac_s = ekat::subview(shoc_input_output.shoc_cldfrac, i); const auto shoc_ql_s = ekat::subview(shoc_input_output.shoc_ql, i); const auto shoc_ql2_s = ekat::subview(shoc_output.shoc_ql2, i); + const auto tkh_out_s = ekat::subview(shoc_output.tkh, i); const auto shoc_mix_s = ekat::subview(shoc_history_output.shoc_mix, i); const auto w_sec_s = ekat::subview(shoc_history_output.w_sec, i); const auto thl_sec_s = ekat::subview(shoc_history_output.thl_sec, i); @@ -675,9 +683,9 @@ Int Functions::shoc_main( wtracer_sfc_s, inv_exner_s, phis_s, // Input workspace, // Workspace host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output - wthv_sec_s, qtracers_s, tk_s, tkh_s, shoc_cldfrac_s, // Input/Output + wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, // Output + pblh_s, shoc_ql2_s, tkh_out_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables @@ -699,9 +707,9 @@ Int Functions::shoc_main( shoc_input.wtracer_sfc, shoc_input.inv_exner, shoc_input.phis, // Input workspace_mgr, // Workspace Manager shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output - shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.tkh, shoc_input_output.shoc_cldfrac, // Input/Output + shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output - shoc_output.pblh, shoc_output.shoc_ql2, // Output + shoc_output.pblh, shoc_output.shoc_ql2, shoc_output.tkh, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables shoc_history_output.wthl_sec, shoc_history_output.wqw_sec, shoc_history_output.wtke_sec, shoc_history_output.uw_sec, shoc_history_output.vw_sec, // Diagnostic Output Variables shoc_history_output.w3, shoc_history_output.wqls_sec, shoc_history_output.brunt, shoc_history_output.isotropy, // Diagnostic Output Variables diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 2d9744b9aed..ccc941bfd05 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -147,8 +147,6 @@ struct Functions view_3d qtracers; // eddy coefficient for momentum [m2/s] view_2d tk; - // eddy coefficient for heat [m2/s] - view_2d tkh; // Cloud fraction [-] view_2d shoc_cldfrac; // cloud liquid mixing ratio [kg/kg] @@ -163,6 +161,8 @@ struct Functions view_1d pblh; // cloud liquid mixing ratio variance [kg^2/kg^2] view_2d shoc_ql2; + // eddy coefficient for heat [m2/s] + view_2d tkh; }; // This struct stores output views for SHOC diagnostics for shoc_main. @@ -890,6 +890,7 @@ struct Functions // Output Variables Scalar& pblh, const uview_1d& shoc_ql2, + const uview_1d& tkh_out, // Diagnostic Output Variables const uview_1d& shoc_mix, const uview_1d& w_sec, @@ -961,6 +962,7 @@ struct Functions // Output Variables const view_1d& pblh, const view_2d& shoc_ql2, + const view_2d& tkh_out, // Diagnostic Output Variables const view_2d& shoc_mix, const view_2d& w_sec, From 7fc6b61f2b555e1e98760313e1e01398e7dfe51f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Apr 2024 16:44:17 -0700 Subject: [PATCH 057/142] Adds a multi process test --- .../mam/eamxx_mam_aci_process_interface.cpp | 4 +- .../multi-process/physics_only/CMakeLists.txt | 1 + .../physics_only/mam/shoc_aci/CMakeLists.txt | 41 ++++++++++ .../physics_only/mam/shoc_aci/input.yaml | 74 +++++++++++++++++++ .../physics_only/mam/shoc_aci/output.yaml | 57 ++++++++++++++ .../tests/single-process/mam/aci/input.yaml | 2 +- .../tests/single-process/mam/aci/output.yaml | 1 + 7 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 840d3eb181a..2d30f9a2bcd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -797,7 +797,7 @@ void MAMAci::set_grids( add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // BALLI:??? - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables @@ -998,7 +998,7 @@ void MAMAci::initialize_impl(const RunType run_type) { liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); // MUST FIXME: This comes from shoc - kvh_ = get_field_in("eddy_diff_mom").get_view(); + kvh_ = get_field_in("eddy_diff_heat").get_view(); nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); niim_ = get_field_out("icenuc_num_immfrz").get_view(); diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 61b224f71ae..046bcecbdec 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -4,6 +4,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(shoc_cld_spa_p3_rrtmgp) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) + add_subdirectory(mam/shoc_aci) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt new file mode 100644 index 00000000000..ea13508e3e5 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_aci) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 scream_rrtmgp mam + LABELS shoc cld p3 rrtmgp physics mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 rrtmgp physics PEM mam + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml new file mode 100644 index 00000000000..bf9f952ab1d --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml @@ -0,0 +1,74 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic,rrtmgp] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + orbital_year: 1990 + do_aerosol_rad: false + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_sens_flux: 0.0 + surf_evap: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + aero_g_sw: 0.0 + aero_ssa_sw: 0.0 + aero_tau_sw: 0.0 + aero_tau_lw: 0.0 + dgnum: 1e-3 + liq_strat_cld_frac: 1e-3 + w_sec: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml new file mode 100644 index 00000000000..a8b22c16ba0 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml @@ -0,0 +1,57 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cld_p3_rrtmgp_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + - T_mid + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index f21bde0ce3f..a4d00685313 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -25,11 +25,11 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - pbl_height : 1.0 phis : 1.0 #These should come from the input file #we should get the following variables from other processes + pbl_height : 1.0 dgnum: 1e-3 eddy_diff_heat: 1e-3 liq_strat_cld_frac: 1e-3 diff --git a/components/eamxx/tests/single-process/mam/aci/output.yaml b/components/eamxx/tests/single-process/mam/aci/output.yaml index cb54b742c5a..670667576c3 100644 --- a/components/eamxx/tests/single-process/mam/aci/output.yaml +++ b/components/eamxx/tests/single-process/mam/aci/output.yaml @@ -6,6 +6,7 @@ Fields: Physics: Field Names: - T_mid + - dgnum output_control: Frequency: 2 From 4f0e0850ca0f9ca9a6716ca7b325a8ce7544bd0b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 4 Apr 2024 10:30:11 -0700 Subject: [PATCH 058/142] Adds two multi-process tests and changes to SHOC for copying tkh_out --- .../eamxx/src/physics/shoc/CMakeLists.txt | 1 + .../shoc/disp/shoc_assign_2d_view_disp.cpp | 29 +++++++++ .../src/physics/shoc/impl/shoc_main_impl.hpp | 5 +- .../eamxx/src/physics/shoc/shoc_functions.hpp | 7 +++ .../multi-process/physics_only/CMakeLists.txt | 3 +- .../shoc_cldfrac_mam4_aci_p3/CMakeLists.txt | 41 +++++++++++++ .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 61 +++++++++++++++++++ .../mam/shoc_cldfrac_mam4_aci_p3/output.yaml | 44 +++++++++++++ .../CMakeLists.txt | 2 +- .../input.yaml | 0 .../output.yaml | 0 11 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml rename components/eamxx/tests/multi-process/physics_only/mam/{shoc_aci => shoc_cldfrac_mam4_aci_p3_rrtmgp}/CMakeLists.txt (96%) rename components/eamxx/tests/multi-process/physics_only/mam/{shoc_aci => shoc_cldfrac_mam4_aci_p3_rrtmgp}/input.yaml (100%) rename components/eamxx/tests/multi-process/physics_only/mam/{shoc_aci => shoc_cldfrac_mam4_aci_p3_rrtmgp}/output.yaml (100%) diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index e37379095d0..f57506fe0ed 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -85,6 +85,7 @@ set(SHOC_SK_SRCS disp/shoc_diag_third_shoc_moments_disp.cpp disp/shoc_assumed_pdf_disp.cpp disp/shoc_update_host_dse_disp.cpp + disp/shoc_assign_2d_view_disp.cpp ) if (NOT SCREAM_DEBUG) diff --git a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp new file mode 100644 index 00000000000..8f5592a3836 --- /dev/null +++ b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp @@ -0,0 +1,29 @@ +#include "shoc_functions.hpp" + +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace shoc { + +template<> +void Functions +::shoc_assign_2d_view_disp( + const Int& shcol, + const Int& nlev, + const view_2d& input_view, + const view_2d& output_view) +{ + /*using ExeSpace = typename KT::ExeSpace; + + const auto nlev_packs = ekat::npack(nlev); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nlev_packs); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const Int i = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { + output_view (i,k) = input_view(i,k); + }); + });*/ +} + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 23348a19f50..34a0455c14f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -311,7 +311,7 @@ void Functions::shoc_main_internal( workspace, // Workspace pblh); // Output - // Assign tkh to tkh output variable + // Assign tkh to the tkh output variable const Int nlev_pack = ekat::npack(nlev); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { tkh_out(k) = tkh(k); @@ -574,7 +574,8 @@ void Functions::shoc_main_internal( workspace_mgr, // Workspace mgr pblh); // Output - tkh_out = tkh; + // Assign tkh to the tkh output variable + shoc_assign_2d_view_disp(shcol, nlev, tkh_out,tkh); } #endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index ccc941bfd05..0b651263064 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -1205,6 +1205,13 @@ struct Functions const view_2d& tk, const view_2d& tkh, const view_2d& isotropy); + + static void shoc_assign_2d_view_disp( + const Int& shcol, + const Int& nlev, + const view_2d& input_view, + const view_2d& output_view); + #endif }; // struct Functions diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 046bcecbdec..8247f178561 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -4,7 +4,8 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(shoc_cld_spa_p3_rrtmgp) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) - add_subdirectory(mam/shoc_aci) + add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) + add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt new file mode 100644 index 00000000000..946d68cb750 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_cldfrac_mam4_aci_p3) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 mam + LABELS shoc cld p3 physics mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 physics PEM mam + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml new file mode 100644 index 00000000000..76efb136c00 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -0,0 +1,61 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_sens_flux: 0.0 + surf_evap: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + dgnum: 1e-3 + liq_strat_cld_frac: 1e-3 + w_sec: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml new file mode 100644 index 00000000000..f4aec357e9f --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml @@ -0,0 +1,44 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cld_p3_rrtmgp_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + - T_mid +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt similarity index 96% rename from components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt index ea13508e3e5..bb30c120d38 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt @@ -1,6 +1,6 @@ INCLUDE (ScreamUtils) -set (TEST_BASE_NAME shoc_aci) +set (TEST_BASE_NAME shoc_cldfrac_mam4_aci_p3_rrtmgp) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml similarity index 100% rename from components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml rename to components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml similarity index 100% rename from components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml rename to components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml From dc5cec22b75e343b6c1b1bdc093f3fe0fa1d57df Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 4 Apr 2024 11:15:30 -0700 Subject: [PATCH 059/142] Adds shoc and aci multi process test --- .../multi-process/physics_only/CMakeLists.txt | 1 + .../mam/shoc_mam4_aci/CMakeLists.txt | 41 ++++++++++++++ .../physics_only/mam/shoc_mam4_aci/input.yaml | 56 +++++++++++++++++++ .../mam/shoc_mam4_aci/output.yaml | 17 ++++++ 4 files changed, 115 insertions(+) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 8247f178561..2b63cd44d37 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -4,6 +4,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(shoc_cld_spa_p3_rrtmgp) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) + add_subdirectory(mam/shoc_mam4_aci) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt new file mode 100644 index 00000000000..af6fff36b0a --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_mam4_aci) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc mam + LABELS shoc physics mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc physics mam + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml new file mode 100644 index 00000000000..d34390e56d5 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -0,0 +1,56 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,mam4_aci] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_sens_flux: 0.0 + surf_evap: 0.0 + dgnum: 1e-3 + liq_strat_cld_frac: 1e-3 + w_sec: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml new file mode 100644 index 00000000000..590cd86ab7e --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml @@ -0,0 +1,17 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cld_p3_rrtmgp_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From ad8ec235bc44df7a36e8c3ed1c1ca54e1d56f4b1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 4 Apr 2024 11:52:39 -0700 Subject: [PATCH 060/142] Adds another multi-process test with optics --- .../multi-process/physics_only/CMakeLists.txt | 1 + .../CMakeLists.txt | 41 +++++++++ .../input.yaml | 83 +++++++++++++++++++ .../output.yaml | 57 +++++++++++++ 4 files changed, 182 insertions(+) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 2b63cd44d37..b9c13234796 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -7,6 +7,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_mam4_aci) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) + add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt new file mode 100644 index 00000000000..c956df2b8fd --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 scream_rrtmgp mam + LABELS shoc cld p3 rrtmgp physics mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 rrtmgp physics PEM mam + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml new file mode 100644 index 00000000000..5bd15fa7d5b --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml @@ -0,0 +1,83 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic, mam4_optics, rrtmgp] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + orbital_year: 1990 + do_aerosol_rad: false + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_sens_flux: 0.0 + surf_evap: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + dgnum: 1e-3 + liq_strat_cld_frac: 1e-3 + w_sec: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml new file mode 100644 index 00000000000..a8b22c16ba0 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml @@ -0,0 +1,57 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cld_p3_rrtmgp_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + - T_mid + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 929da35210ad4c92de0a792f107ac55c6939531f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Apr 2024 12:14:55 -0700 Subject: [PATCH 061/142] Adds w_variance in SHOC --- .../mam/eamxx_mam_aci_process_interface.cpp | 17 ++++++++++++----- .../shoc/eamxx_shoc_process_interface.cpp | 15 ++++++++------- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 4 ++-- .../physics_only/mam/shoc_mam4_aci/input.yaml | 1 - 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 2d30f9a2bcd..193e30ee52e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -786,10 +786,17 @@ void MAMAci::set_grids( m2.set_string("m^2"); auto s2 = s * s; s2.set_string("s^2"); - // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and - // BALLI:??? - // Vertical velocity variance (wp2) at midpoints - add_field("w_sec", scalar3d_layout_int, m2 / s2, grid_name); + + // NOTE: w_variance im microp_aero_run.F90 is at "itim_old" dynamics time step + // Since, we are using SE dycore, itim_old is 1 which is equivalent to the + // current time step. For other dycores (such as EUL), it may be different + // and we might need to revisit this + + //FIXME: w_variance in microp_aero_run.F90 is at the interfaces but + // SHOC provides it at the midpoints. Verify how it is being used. + + // Vertical velocity variance at midpoints + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // BALLI:??? // FIXME:liq_strat_cld_frac may also need OLD time @@ -991,7 +998,7 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { } void MAMAci::initialize_impl(const RunType run_type) { - w_sec_ = get_field_in("w_sec").get_view(); + w_sec_ = get_field_in("w_variance").get_view(); // MUST FIXME: is it an input, should we invoke calcsize here?? dgnum_ = get_field_in("dgnum").get_view(); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index a988948c0f1..70404b7997a 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -80,17 +80,18 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables add_field("pbl_height", scalar2d_layout_col, m, grid_name); add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("w_variance", scalar3d_layout_mid, m2/s2, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -338,7 +339,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; history_output.isotropy = m_buffer.isotropy; - history_output.w_sec = m_buffer.w_sec; + history_output.w_sec = get_field_out("w_variance").get_view(); history_output.thl_sec = m_buffer.thl_sec; history_output.qw_sec = m_buffer.qw_sec; history_output.qwthl_sec = m_buffer.qwthl_sec; diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index 76efb136c00..ca0a99c50cb 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -52,8 +52,8 @@ initial_conditions: precip_ice_surf_mass: 0.0 precip_liq_surf_mass: 0.0 dgnum: 1e-3 - liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 + #liq_strat_cld_frac: 1e-3 + #w_sec: 1e-3 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml index d34390e56d5..84212175a3d 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -48,7 +48,6 @@ initial_conditions: surf_evap: 0.0 dgnum: 1e-3 liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 # The parameters for I/O control Scorpio: From 0e2b470a66f8e156636569917e2f2b2309e34d32 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Apr 2024 17:42:53 -0700 Subject: [PATCH 062/142] Add old and new cloud fraction inputs from SHOC --- .../mam/eamxx_mam_aci_process_interface.cpp | 46 ++++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 + .../shoc/eamxx_shoc_process_interface.cpp | 12 +++-- .../shoc/eamxx_shoc_process_interface.hpp | 10 +++- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 8 +++- .../physics_only/mam/shoc_mam4_aci/input.yaml | 5 +- 6 files changed, 52 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 193e30ee52e..10e41db4e43 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -299,13 +299,11 @@ void compute_nucleate_ice_tendencies( }); } KOKKOS_INLINE_FUNCTION -void store_liquid_cloud_fraction(const haero::ThreadTeam &team, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - MAMAci::const_view_2d qc, - MAMAci::const_view_2d qi, - MAMAci::const_view_2d liqcldf, const int icol, - const int top_lev) { +void store_liquid_cloud_fraction( + const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, MAMAci::const_view_2d qc, + MAMAci::const_view_2d qi, MAMAci::const_view_2d liqcldf, + MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev) { //------------------------------------------------------------- // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value @@ -323,8 +321,7 @@ void store_liquid_cloud_fraction(const haero::ThreadTeam &team, const Real qcld = qc(icol, kk) + qi(icol, kk); if(qcld > qsmall) { cloud_frac_new(icol, kk) = liqcldf(icol, kk); - cloud_frac_old(icol, kk) = - liqcldf(icol, kk); // FIXME should be liqcldf_old + cloud_frac_old(icol, kk) = liqcldf_prev(icol, kk); } else { cloud_frac_new(icol, kk) = 0; cloud_frac_old(icol, kk) = 0; @@ -336,6 +333,7 @@ void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d cloud_frac_old, mam_coupling::WetAtmosphere &wet_atmosphere, MAMAci::const_view_2d liqcldf, + MAMAci::const_view_2d liqcldf_prev, const int top_lev) { MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d qi = wet_atmosphere.qi; @@ -343,7 +341,7 @@ void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, - qi, liqcldf, icol, top_lev); + qi, liqcldf, liqcldf_prev, icol, top_lev); }); } KOKKOS_INLINE_FUNCTION @@ -786,22 +784,27 @@ void MAMAci::set_grids( m2.set_string("m^2"); auto s2 = s * s; s2.set_string("s^2"); - + // NOTE: w_variance im microp_aero_run.F90 is at "itim_old" dynamics time step - // Since, we are using SE dycore, itim_old is 1 which is equivalent to the + // Since, we are using SE dycore, itim_old is 1 which is equivalent to the // current time step. For other dycores (such as EUL), it may be different // and we might need to revisit this - //FIXME: w_variance in microp_aero_run.F90 is at the interfaces but - // SHOC provides it at the midpoints. Verify how it is being used. + // FIXME: w_variance in microp_aero_run.F90 is at the interfaces but + // SHOC provides it at the midpoints. Verify how it is being used. // Vertical velocity variance at midpoints add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); - // BALLI:??? - // FIXME:liq_strat_cld_frac may also need OLD time - // Liquid stratiform cloud fraction at midpoints - add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, + // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is + // equivalent + // to "alst" in the shoc_intr.F90. In the C++ code, it is used as + // "shoc_cldfrac" and in the F90 code it is called "cloud_frac" + + // Liquid stratiform cloud fraction at midpoints + add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name); + + add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, grid_name); // BALLI:??? add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, @@ -1001,8 +1004,9 @@ void MAMAci::initialize_impl(const RunType run_type) { w_sec_ = get_field_in("w_variance").get_view(); // MUST FIXME: is it an input, should we invoke calcsize here?? - dgnum_ = get_field_in("dgnum").get_view(); - liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + dgnum_ = get_field_in("dgnum").get_view(); + liqcldf_ = get_field_in("cldfrac_liq").get_view(); + liqcldf_prev_ = get_field_in("cldfrac_liq_prev").get_view(); // MUST FIXME: This comes from shoc kvh_ = get_field_in("eddy_diff_heat").get_view(); @@ -1192,7 +1196,7 @@ void MAMAci::run_impl(const double dt) { dry_atm_, aitken_dry_dia_, nlev_); store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, - wet_atm_, liqcldf_, top_lev_); + wet_atm_, liqcldf_, liqcldf_prev_, top_lev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index a73e04b8575..6873ccd3397 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -65,6 +65,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d naai_hom_; view_2d naai_; const_view_2d liqcldf_; + const_view_2d liqcldf_prev_; const_view_2d kvh_; view_2d cloud_frac_new_; diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 70404b7997a..6059a4369ed 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -88,10 +88,11 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d_layout_col, m, grid_name); - add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); - add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("w_variance", scalar3d_layout_mid, m2/s2, grid_name, ps); + add_field("pbl_height", scalar2d_layout_col, m, grid_name); + add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("w_variance", scalar3d_layout_mid, m2/s2, grid_name, ps); + add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -257,6 +258,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto& qv = get_field_out("qv").get_view(); const auto& tke = get_field_out("tke").get_view(); const auto& cldfrac_liq = get_field_out("cldfrac_liq").get_view(); + const auto& cldfrac_liq_prev = get_field_out("cldfrac_liq_prev").get_view(); const auto& sgs_buoy_flux = get_field_out("sgs_buoy_flux").get_view(); const auto& tk = get_field_out("eddy_diff_mom").get_view(); const auto& inv_qc_relvar = get_field_out("inv_qc_relvar").get_view(); @@ -301,7 +303,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) T_mid,p_mid,p_int,pseudo_density,omega,phis,surf_sens_flux,surf_evap, surf_mom_flux,qtracers,qv,qc,qc_copy,tke,tke_copy,z_mid,z_int, dse,rrho,rrho_i,thv,dz,zt_grid,zi_grid,wpthlp_sfc,wprtp_sfc,upwp_sfc,vpwp_sfc, - wtracer_sfc,wm_zt,inv_exner,thlm,qw); + wtracer_sfc,wm_zt,inv_exner,thlm,qw, cldfrac_liq, cldfrac_liq_prev); // Input Variables: input.zt_grid = shoc_preprocess.zt_grid; diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index d3f5b05f88d..1bc0d14afe0 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -83,6 +83,9 @@ class SHOCMacrophysics : public scream::AtmosphereProcess Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { + + cldfrac_liq_prev(i,k)=cldfrac_liq(i,k); + const auto range = ekat::range(k*Spack::n); const Smask in_nlev_range = (range < nlev); @@ -198,6 +201,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess view_2d thlm; view_2d qw; view_2d cloud_frac; + view_2d cldfrac_liq; + view_2d cldfrac_liq_prev; // Assigning local variables void set_variables(const int ncol_, const int nlev_, const int num_qtracers_, @@ -213,7 +218,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const view_2d& dse_, const view_2d& rrho_, const view_2d& rrho_i_, const view_2d& thv_, const view_2d& dz_,const view_2d& zt_grid_,const view_2d& zi_grid_, const view_1d& wpthlp_sfc_, const view_1d& wprtp_sfc_,const view_1d& upwp_sfc_,const view_1d& vpwp_sfc_, const view_2d& wtracer_sfc_, - const view_2d& wm_zt_,const view_2d& inv_exner_,const view_2d& thlm_,const view_2d& qw_) + const view_2d& wm_zt_,const view_2d& inv_exner_,const view_2d& thlm_,const view_2d& qw_, + const view_2d& cldfrac_liq_, const view_2d& cldfrac_liq_prev_) { ncol = ncol_; nlev = nlev_; @@ -254,6 +260,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess inv_exner = inv_exner_; thlm = thlm_; qw = qw_; + cldfrac_liq=cldfrac_liq_; + cldfrac_liq_prev=cldfrac_liq_prev_; } // set_variables }; // SHOCPreprocess /* --------------------------------------------------------------------------------------------*/ diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index ca0a99c50cb..f41b961a4f5 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -47,13 +47,17 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variable required for shoc surf_sens_flux: 0.0 surf_evap: 0.0 + + #variable required for p3 precip_ice_surf_mass: 0.0 precip_liq_surf_mass: 0.0 + + #variables required for mam4_aci dgnum: 1e-3 - #liq_strat_cld_frac: 1e-3 - #w_sec: 1e-3 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml index 84212175a3d..1d6d4f48eba 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -44,10 +44,13 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables needed for SHOC surf_sens_flux: 0.0 surf_evap: 0.0 + + #variables needed for mam4-aci dgnum: 1e-3 - liq_strat_cld_frac: 1e-3 # The parameters for I/O control Scorpio: From 968f5f3b5708816ad6a4b4dbb4e87f598f62aa66 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 9 Apr 2024 15:57:15 -0700 Subject: [PATCH 063/142] A lot of cleanup and changed Fieldlayour for dgnum --- .../mam/eamxx_mam_aci_process_interface.cpp | 188 +++++++++--------- .../eamxx/src/share/field/field_tag.hpp | 38 ++++ .../input.yaml | 8 +- .../input.yaml | 10 +- .../tests/single-process/mam/aci/input.yaml | 6 +- 5 files changed, 145 insertions(+), 105 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 10e41db4e43..5b9a2b1d2f0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -210,7 +210,7 @@ void compute_aitken_dry_diameter(const haero::ThreadTeam &team, const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol, kk) = dgnum(icol, kk, aitken_idx); + aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); }); } void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, @@ -230,7 +230,7 @@ void compute_nucleate_ice_tendencies( MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, mam_coupling::AerosolState &dry_aerosol_state, mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d aitken_dry_dia, - const int nlev) { + const int nlev, const double dt) { //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation @@ -248,17 +248,13 @@ void compute_nucleate_ice_tendencies( // Set up an atmosphere, surface, diagnostics, pronostics and tendencies // class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), - ekat::subview(qv_dry, icol), dummy, dummy, dummy, dummy, dummy, - dummy, dummy, ekat::subview(cldfrac, icol), - ekat::subview(w_updraft, icol), pblh); - // set surface state data haero::Surface surf{}; mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + // nucleation doesn't use any diagnostics, so it's okay to leave this // alone for now mam4::Diagnostics diags(nlev); @@ -283,7 +279,7 @@ void compute_nucleate_ice_tendencies( // values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; - const Real t = 0, dt = 0; + const Real t = 0; /* NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". Only output we care @@ -294,8 +290,8 @@ void compute_nucleate_ice_tendencies( ! input rho, wsubice, strat_cld_frac, dgnum, & ! input naai, naai_hom) ! output */ - nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, - progs, diags, tends); + nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, + surf, progs, diags, tends); }); } KOKKOS_INLINE_FUNCTION @@ -605,12 +601,6 @@ void call_hetfrz_compute_tendencies( haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); - /*Real pblh = 0; - haero::Atmosphere atmos( - nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, - ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, - dummy, dummy, dummy, dummy, dummy, pblh);*/ - // set surface state data haero::Surface surf{}; mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aerosol_state, icol); @@ -707,16 +697,14 @@ MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) void MAMAci::set_grids( const std::shared_ptr grids_manager) { grid_ = grids_manager->get_grid("Physics"); // Use physics grid - const auto &grid_name = grid_->name(); + const auto &grid_name = grid_->name(); // Name of the grid ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column // Allocate memory for the class members // Kokkos::resize only works on host to allocates memory - Kokkos::resize( - rho_, ncol_, - nlev_); // FIXME: rho_ is only used internally in compute_w0_and_rho + Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -735,60 +723,69 @@ void MAMAci::set_grids( using namespace ShortFieldTagsNames; // Layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // mid points - FieldLayout scalar3d_layout_int{{COL, ILEV}, - {ncol_, nlev_ + 1}}; // interfaces + // mid points + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + // interfaces + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); + auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); + auto nondim = ekat::units::Units::nondimensional(); + // atmospheric quantities - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity [kg/kg] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid number mixing ratio [1/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud ice number mixing ratio [1/kg] - - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // Temperature[K] at midpoints + // specific humidity [kg/kg] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // Total pressure [Pa] at midpoints - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // Total pressure [Pa] at interfaces - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // Layer thickness(pdel) [Pa] at midpoints + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_layout_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); - add_field("pbl_height", scalar2d_layout_col, m, - grid_name); // planetary boundary layer height + // planetary boundary layer height + add_field("pbl_height", scalar2d_layout_col, m, grid_name); // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); - // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM - // These outputs should come from the cloud macrophysics process (e.g., SHOC) auto m2 = m * m; m2.set_string("m^2"); auto s2 = s * s; s2.set_string("s^2"); - // NOTE: w_variance im microp_aero_run.F90 is at "itim_old" dynamics time step - // Since, we are using SE dycore, itim_old is 1 which is equivalent to the - // current time step. For other dycores (such as EUL), it may be different - // and we might need to revisit this + // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time + // step Since, we are using SE dycore, itim_old is 1 which is equivalent to + // the current time step. For other dycores (such as EUL), it may be different + // and we might need to revisit this. // FIXME: w_variance in microp_aero_run.F90 is at the interfaces but // SHOC provides it at the midpoints. Verify how it is being used. @@ -797,26 +794,26 @@ void MAMAci::set_grids( add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is - // equivalent - // to "alst" in the shoc_intr.F90. In the C++ code, it is used as - // "shoc_cldfrac" and in the F90 code it is called "cloud_frac" + // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as + // "shoc_cldfrac" and in the F90 code it is called "cloud_frac" // Liquid stratiform cloud fraction at midpoints add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name); + // Previous value of liquid stratiform cloud fraction at midpoints add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, grid_name); - // BALLI:??? - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, - grid_name); // Eddy diffusivity for heat + + // Eddy diffusivity for heat + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ - {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points - // BALLI:??? - add_field("dgnum", scalar4d_layout_mid, m, - grid_name); // dry diameter of aerosols + FieldLayout scalar4d_layout_mid{{COL, NMODES, LEV}, + {ncol_, num_aero_modes, nlev_}}; + + // dry diameter of aerosols [m] + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // ======================================================================== // Output from this whole process @@ -895,56 +892,50 @@ void MAMAci::set_grids( // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ - // BALLI:??? - // FIXME: THis looks like an internal variable for dropmixnuc, why we need it - // here??? - add_field("qcld", scalar3d_layout_mid, n_unit, - grid_name); // cloud droplet number mixing ratio [#/kg] - // BALLI:??? - // FIXME:This array should update the mmrs and nmrs - // tendencies for interstitial and cloud borne aerosols [#/kg] - add_field( - "ptend_q", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::aero_model::pcnst}}, - n_unit, grid_name); + + constexpr int pcnst = mam4::aero_model::pcnst; + FieldLayout scalar4d_layout_nconst_mid{{COL, LEV, MAM_NCNST}, + {ncol_, nlev_, pcnst}}; + + // tendencies for interstitial and cloud borne aerosols [kg/kg or #/kg] + add_field("ptend_q", scalar4d_layout_nconst_mid, q_unit, grid_name); // tendency in droplet number mixing ratio [#/kg/s] add_field("tendnd", scalar3d_layout_mid, n_unit / s, grid_name); // activation fraction for aerosol number [fraction] - add_field( - "factnum", - FieldLayout{{COL, NMODES, LEV}, - {ncol_, mam_coupling::num_aero_modes(), nlev_}}, - nondim, grid_name); + add_field("factnum", scalar4d_layout_mid, nondim, grid_name); + + // NOTE: Here is a series of internal dropmixnuc variables; + // maybe we should move them to diagnostics later + + // cloud droplet number mixing ratio [#/kg] + add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); auto inv_m2 = 1 / m / m; inv_m2.set_string("#/m2"); - // BALLI:??? FIXME: This is internal diagnostic variable + // column-integrated droplet number [#/m2] - add_field("ndropcol", scalar3d_layout_mid, inv_m2, - grid_name); // - // BALLI:??? FIXME: This is internal diagnostic variable + add_field("ndropcol", scalar3d_layout_mid, inv_m2, grid_name); + // droplet number mixing ratio tendency due to mixing [#/kg/s] add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable + // droplet number mixing ratio source tendency [#/kg/s] add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable // subgrid vertical velocity [m/s] add_field("wtke", scalar3d_layout_mid, m / s, grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable // number conc of aerosols activated at supersat [#/m^3] - // note: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud - // [#/m^2/s]] - // / [aero. number conc. in updraft, just below - // cloudbase [#/m^3]] - add_field( - "ccn", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, - m3_inv, grid_name); + // NOTE: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] + // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] + + constexpr int psat = mam4::ndrop::psat; + FieldLayout scalar4d_layout_psat_mid{{COL, LEV, MAM_PSAT}, + {ncol_, nlev_, psat}}; + add_field("ccn", scalar4d_layout_psat_mid, m3_inv, grid_name); // BALLI:??? FIXME: This is internal diagnostic variable // column tendency for diagnostic output @@ -1174,6 +1165,7 @@ void MAMAci::run_impl(const double dt) { // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion + // FIXME: Why wet_atm_ used here????? We should use dry_atm compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, wet_atm_, dry_atm_, top_lev_, nlev_); @@ -1193,8 +1185,8 @@ void MAMAci::run_impl(const double dt) { // FIXME: Find out in-outs of the following call! compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, dry_aero_, - dry_atm_, aitken_dry_dia_, nlev_); - + dry_atm_, aitken_dry_dia_, nlev_, dt); + // FIXME: Why wet_atm_ used here????? We should use dry_atm store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, wet_atm_, liqcldf_, liqcldf_prev_, top_lev_); @@ -1232,7 +1224,7 @@ void MAMAci::run_impl(const double dt) { team_policy, coltend_outp_, coltend_, nlev_); copy_mam4xx_array_to_scream( team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - + // FIXME: Why wet_atm_ used here????? We should use dry_atm call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, factnum_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index 673208d6ae8..79515420045 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -34,6 +34,14 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, + num_modes, + num_constituents, + psat, + // + nrefindex_real, + nrefindex_im, + ncoef_number, + mode }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -50,6 +58,15 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; + constexpr auto NMODES = FieldTag::num_modes; + constexpr auto MAM_NCNST = FieldTag::num_constituents; + constexpr auto MAM_PSAT = FieldTag::psat; + // + constexpr auto NREFINDEX_REAL = FieldTag::nrefindex_real; + constexpr auto NREFINDEX_IM = FieldTag::nrefindex_im; + + constexpr auto NCOEF_NUMBER = FieldTag::ncoef_number; + constexpr auto MODE = FieldTag::mode; } inline std::string e2str (const FieldTag ft) { @@ -80,6 +97,27 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; + case FieldTag::num_modes: + name = "num_modes"; + break; + case FieldTag::num_constituents: + name = "num_constituents"; + break; + case FieldTag::psat: + name = "psat"; + break; + case FieldTag::nrefindex_real: + name = "refindex_real"; + break; + case FieldTag::nrefindex_im: + name = "refindex_im"; + break; + case FieldTag::ncoef_number: + name = "coef_number"; + break; + case FieldTag::mode: + name = "mode"; + break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml index 5bd15fa7d5b..f3198b1308a 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml @@ -69,13 +69,17 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables required for shoc surf_sens_flux: 0.0 surf_evap: 0.0 + + #variables required for p3 precip_ice_surf_mass: 0.0 precip_liq_surf_mass: 0.0 + + #variables required for mam4_aci dgnum: 1e-3 - liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml index bf9f952ab1d..d61f1318f8f 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml @@ -56,17 +56,23 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables required for shoc surf_sens_flux: 0.0 surf_evap: 0.0 + + #variables required for p3 precip_ice_surf_mass: 0.0 precip_liq_surf_mass: 0.0 + + #variables required for rrtmgp aero_g_sw: 0.0 aero_ssa_sw: 0.0 aero_tau_sw: 0.0 aero_tau_lw: 0.0 + + #variables required for mam4_aci dgnum: 1e-3 - liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index a4d00685313..dc1b779875a 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -32,9 +32,9 @@ initial_conditions: pbl_height : 1.0 dgnum: 1e-3 eddy_diff_heat: 1e-3 - liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 - w_updraft: 1e-3 + cldfrac_liq: 1e-3 + cldfrac_liq_prev: 0.1 + w_variance: 1e-3 # The parameters for I/O control Scorpio: From 5f152af650b8a4d7006fe3efcad4d8906e09daff Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 9 Apr 2024 18:01:01 -0700 Subject: [PATCH 064/142] Clean up round for initialize impl --- .../mam/eamxx_mam_aci_process_interface.cpp | 179 +++++++++--------- .../eamxx/src/share/field/field_tag.hpp | 5 + 2 files changed, 98 insertions(+), 86 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 5b9a2b1d2f0..b6dfbacc45d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -693,9 +693,12 @@ const Real wsubmin = 1; MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) {} -// set grid for all the inputs and outputs +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMAci::set_grids( const std::shared_ptr grids_manager) { + // set grid for all the inputs and outputs grid_ = grids_manager->get_grid("Physics"); // Use physics grid const auto &grid_name = grid_->name(); // Name of the grid @@ -853,18 +856,19 @@ void MAMAci::set_grids( add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } - } - } + } // end for loop num species + } // end for loop for num modes + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } + } // end for loop num gases // ------------------------------------------------------------------------ // Output from ice nucleation process // ------------------------------------------------------------------------ - auto m3_inv = 1 / m / m / m; // inverse of m3 + const auto m3_inv = 1 / m / m / m; // inverse of m3 // number conc of ice nuclei due to heterogeneous freezing [1/m3] add_field("icenuc_num_hetfrz", scalar3d_layout_mid, m3_inv, grid_name); @@ -875,7 +879,7 @@ void MAMAci::set_grids( // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] add_field("icenuc_num_depnuc", scalar3d_layout_mid, m3_inv, - grid_name); // + grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] add_field("icenuc_num_meydep", scalar3d_layout_mid, m3_inv, @@ -927,36 +931,37 @@ void MAMAci::set_grids( // subgrid vertical velocity [m/s] add_field("wtke", scalar3d_layout_mid, m / s, grid_name); + constexpr int psat = mam4::ndrop::psat; + FieldLayout scalar4d_layout_psat_mid{{COL, LEV, MAM_PSAT}, + {ncol_, nlev_, psat}}; // number conc of aerosols activated at supersat [#/m^3] // NOTE: activation fraction fluxes are defined as // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] - - constexpr int psat = mam4::ndrop::psat; - FieldLayout scalar4d_layout_psat_mid{{COL, LEV, MAM_PSAT}, - {ncol_, nlev_, psat}}; add_field("ccn", scalar4d_layout_psat_mid, m3_inv, grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable + constexpr int num_aero_const = mam4::ndrop::ncnst_tot; + FieldLayout scalar4d_layout_naero_const_mid{{COL, LEV, MAM_AERO_NCNST}, + {ncol_, nlev_, num_aero_const}}; + // column tendency for diagnostic output - add_field( - "coltend", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - nondim, grid_name); + add_field("coltend", scalar4d_layout_naero_const_mid, nondim, + grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable // column tendency - add_field( - "coltend_cw", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - nondim, grid_name); + add_field("coltend_cw", scalar4d_layout_naero_const_mid, nondim, + grid_name); - auto cm = m / 100; + const auto cm = m / 100; // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); + // ------------------------------------------------------------------------ + // Output from hetrozenous freezing + // ------------------------------------------------------------------------ + // heterogeous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); @@ -968,16 +973,11 @@ void MAMAci::set_grids( // heterogeous freezing by deposition nucleation [cm^-3 s^-1] add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); - // - /* - * NOTE on other inputs for the aci process: - * 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density - * 2. geopotential height at midpoints: computed geopotential height at - * interfaces, which inturn is computed using pseudo_density, p_mid, T_mid and - * qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). - * qv_mid can be obtained from "get_field_in" call - */ -} // set_grids ends +} // function set_grids ends + +// ================================================================ +// INIT_BUFFERS +// ================================================================ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( @@ -989,18 +989,59 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMMicrophysics."); -} +} // function init_buffers ends +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMAci::initialize_impl(const RunType run_type) { - w_sec_ = get_field_in("w_variance").get_view(); - - // MUST FIXME: is it an input, should we invoke calcsize here?? + // ------------------------------------------------------------------------ + // Input fields read in from IC file, namelist or other processes + // ------------------------------------------------------------------------ + w_sec_ = get_field_in("w_variance").get_view(); dgnum_ = get_field_in("dgnum").get_view(); liqcldf_ = get_field_in("cldfrac_liq").get_view(); liqcldf_prev_ = get_field_in("cldfrac_liq_prev").get_view(); + kvh_ = get_field_in("eddy_diff_heat").get_view(); + + // store fields only to be converted to dry mmrs in wet_atm_ + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + // store rest fo the atm fields in dry_atm_in + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - // MUST FIXME: This comes from shoc - kvh_ = get_field_in("eddy_diff_heat").get_view(); + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // pbl_height + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + + // geometric thickness of layers (m) + dry_atm_.dz = buffer_.dz; + + // geopotential height above surface at interface levels (m) + dry_atm_.z_iface = buffer_.z_iface; + + // geopotential height above surface at mid levels (m) + dry_atm_.z_mid = buffer_.z_mid; + + // total cloud fraction + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + + // computed updraft velocity + dry_atm_.w_updraft = buffer_.w_updraft; nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); niim_ = get_field_out("icenuc_num_immfrz").get_view(); @@ -1008,6 +1049,7 @@ void MAMAci::initialize_impl(const RunType run_type) { nimey_ = get_field_out("icenuc_num_meydep").get_view(); naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); + naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); qcld_ = get_field_out("qcld").get_view(); ptend_q_output_ = get_field_out("ptend_q").get_view(); @@ -1020,36 +1062,6 @@ void MAMAci::initialize_impl(const RunType run_type) { ccn_ = get_field_out("ccn").get_view(); coltend_outp_ = get_field_out("coltend").get_view(); coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; - - // pbl_height from SHOC - dry_atm_.pblh = get_field_in("pbl_height").get_view(); - - dry_atm_.dz = buffer_.dz; // geometric thickness of layers (m) - dry_atm_.z_iface = buffer_.z_iface; // geopotential height above - // surface at interface levels (m) - dry_atm_.z_mid = - buffer_.z_mid; // geopotential height above surface at mid levels (m) - - dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); - dry_atm_.w_updraft = buffer_.w_updraft; - hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); hetfrz_contact_nucleation_tend_ = @@ -1057,14 +1069,6 @@ void MAMAci::initialize_impl(const RunType run_type) { hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); - // allocate work - // FIXME: store 25 into a const var or get from MAM4xx - for(int icnst = 0; icnst < 25; ++icnst) { - qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); - } - // FIXME :store 40 in a const int or get from MAM4xx - state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, 40); - // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -1089,9 +1093,6 @@ void MAMAci::initialize_impl(const RunType run_type) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; - std::cout << m << ":" << a << ":" << int_mmr_field_name << ":" - << dry_aero_.int_aero_mmr[m][a](0, 0) << ":" - << wet_aero_.int_aero_mmr[m][a](0, 0) << std::endl; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -1111,6 +1112,13 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // Allocate work arrays + for(int icnst = 0; icnst < mam4::ndrop::ncnst_tot; ++icnst) { + qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); + } + state_q_work_ = + view_3d("state_q_work_", ncol_, nlev_, mam4::aero_model::pcnst); + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. @@ -1118,16 +1126,16 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - Kokkos::resize(ptend_q_[i], ncol_, nlev_); // MUST FIXME:Do we need this? + Kokkos::resize(ptend_q_[i], ncol_, nlev_); } for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { // MUST FIXME:store 2 in a const variable + for(int j = 0; j < 2; ++j) { Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); } } - for(int i = 0; i < 42; ++i) // MUST FIXME:store 42 in a const var + for(int i = 0; i < 42; ++i) Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); // nact : fractional aero. number activation rate [/s] @@ -1147,11 +1155,12 @@ void MAMAci::initialize_impl(const RunType run_type) { // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); -} +} // end function initialize_impl +// ================================================================ +// RUN_IMPL +// ================================================================ void MAMAci::run_impl(const double dt) { - m_atm_logger->log(ekat::logger::LogLevel::info, "calling ACI run"); - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -1162,8 +1171,6 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - // All the inputs are available to compute w0 and rho - // Convert from omega to w (vertical velocity) // Negative omega means rising motion // FIXME: Why wet_atm_ used here????? We should use dry_atm compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, wet_atm_, diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index 79515420045..7e6b0b83412 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -36,6 +36,7 @@ enum class FieldTag { TimeLevel, num_modes, num_constituents, + num_aero_const, psat, // nrefindex_real, @@ -60,6 +61,7 @@ namespace ShortFieldTagsNames { constexpr auto CMP = FieldTag::Component; constexpr auto NMODES = FieldTag::num_modes; constexpr auto MAM_NCNST = FieldTag::num_constituents; + constexpr auto MAM_AERO_NCNST = FieldTag::num_aero_const; constexpr auto MAM_PSAT = FieldTag::psat; // constexpr auto NREFINDEX_REAL = FieldTag::nrefindex_real; @@ -103,6 +105,9 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::num_constituents: name = "num_constituents"; break; + case FieldTag::num_aero_const: + name = "num_aero_const"; + break; case FieldTag::psat: name = "psat"; break; From 9ce18afbe35907d3d10ca7c3d404990d6d7eeb76 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 9 Apr 2024 18:18:49 -0700 Subject: [PATCH 065/142] Moves oemga to DryAtmosphere --- .../mam/eamxx_mam_aci_process_interface.cpp | 14 +++++++------- .../eamxx_mam_microphysics_process_interface.cpp | 3 ++- .../mam/eamxx_mam_optics_process_interface.cpp | 3 ++- components/eamxx/src/physics/mam/mam_coupling.hpp | 4 ++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b6dfbacc45d..e6d0fb5f8f7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -63,7 +63,7 @@ void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, const int top_lev, const int nlev) { // Get physical constants using C = physics::Constants; - MAMAci::const_view_2d omega = wet_atmosphere.omega; + MAMAci::const_view_2d omega = dry_atmosphere.omega; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for( @@ -1005,18 +1005,18 @@ void MAMAci::initialize_impl(const RunType run_type) { kvh_ = get_field_in("eddy_diff_heat").get_view(); // store fields only to be converted to dry mmrs in wet_atm_ - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); // store rest fo the atm fields in dry_atm_in dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.qv = buffer_.qv_dry; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 8bc727c325e..6ae36835302 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -201,7 +201,7 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { wet_atm_.nc = get_field_out("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); @@ -209,6 +209,7 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.dz = buffer_.dz; dry_atm_.z_iface = buffer_.z_iface; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp index a55bf84ca17..442a6500f21 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -140,7 +140,7 @@ void MAMOptics::initialize_impl(const RunType run_type) { wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + constexpr int ntot_amode = mam4::AeroConfig::num_modes(); @@ -153,6 +153,7 @@ void MAMOptics::initialize_impl(const RunType run_type) { .get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.dz = buffer_.dz; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 90b8a50ea48..6b5cfbd186d 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -260,7 +260,6 @@ struct WetAtmosphere { const_view_2d nc; // wet cloud liquid water number mixing ratio [# / kg moist air] const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice water / kg moist air] const_view_2d ni; // wet cloud ice water number mixing ratio [# / kg moist air] - const_view_2d omega; // vertical pressure velocity [Pa/s] }; // This type stores multi-column views related to the dry atmospheric state @@ -283,6 +282,7 @@ struct DryAtmosphere { view_2d w_updraft; // updraft velocity [m/s] const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] + const_view_2d omega; // vertical pressure velocity [Pa/s] }; // This type stores aerosol number and mass mixing ratios evolved by MAM. It @@ -625,7 +625,7 @@ void compute_updraft_velocities(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { dry_atm.dz(i,k) = PF::calculate_dz(dry_atm.p_del(i,k), dry_atm.p_mid(i,k), dry_atm.T_mid(i,k), wet_atm.qv(i,k)); const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); - dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); + dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(dry_atm.omega(i,k), rho); }); } From 5c1c7a68ee1cf56a2f5953aca63589e59c722f04 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 10 Apr 2024 17:48:25 -0700 Subject: [PATCH 066/142] Fixes cld frac zero out bug and removes wet_atm_ from run_impl --- .../mam/eamxx_mam_aci_process_interface.cpp | 96 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- 2 files changed, 45 insertions(+), 55 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e6d0fb5f8f7..4a7f438e89f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -58,7 +58,6 @@ void compute_w0_and_rho(const haero::ThreadTeam &team, MAMAci::view_2d w0, } void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, MAMAci::view_2d rho, - mam_coupling::WetAtmosphere &wet_atmosphere, mam_coupling::DryAtmosphere &dry_atmosphere, const int top_lev, const int nlev) { // Get physical constants @@ -296,48 +295,42 @@ void compute_nucleate_ice_tendencies( } KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, MAMAci::const_view_2d qc, + const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac, + MAMAci::view_2d cloud_frac_prev, MAMAci::const_view_2d qc, MAMAci::const_view_2d qi, MAMAci::const_view_2d liqcldf, - MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev) { + MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, + const int nlev) { //------------------------------------------------------------- // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value - - // MUST FIXME NOTE: We need old and new liquid cloud fractions here. - // We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to - // store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we - // will make a note of it and use the new cloud fraction for the old cloud - // fraction. //------------------------------------------------------------- - static constexpr auto qsmall = - 1e-18; // cut-off for cloud amount (ice or liquid) + // cut-off for cloud amount (ice or liquid) + static constexpr auto qsmall = 1e-18; Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - const Real qcld = qc(icol, kk) + qi(icol, kk); - if(qcld > qsmall) { - cloud_frac_new(icol, kk) = liqcldf(icol, kk); - cloud_frac_old(icol, kk) = liqcldf_prev(icol, kk); + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + if((qc(icol, kk) + qi(icol, kk)) > qsmall) { + cloud_frac(icol, kk) = liqcldf(icol, kk); + cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); } else { - cloud_frac_new(icol, kk) = 0; - cloud_frac_old(icol, kk) = 0; + cloud_frac(icol, kk) = 0; + cloud_frac_prev(icol, kk) = 0; } }); } void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::view_2d cloud_frac, + MAMAci::view_2d cloud_frac_prev, + mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::const_view_2d liqcldf, MAMAci::const_view_2d liqcldf_prev, - const int top_lev) { - MAMAci::const_view_2d qc = wet_atmosphere.qc; - MAMAci::const_view_2d qi = wet_atmosphere.qi; + const int top_lev, const int nlev) { + MAMAci::const_view_2d qc = dry_atmosphere.qc; + MAMAci::const_view_2d qi = dry_atmosphere.qi; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, - qi, liqcldf, liqcldf_prev, icol, top_lev); + store_liquid_cloud_fraction(team, cloud_frac, cloud_frac_prev, qc, qi, + liqcldf, liqcldf_prev, icol, top_lev, nlev); }); } KOKKOS_INLINE_FUNCTION @@ -374,8 +367,8 @@ void call_function_dropmixnuc( MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, MAMAci::view_2d rpdel, MAMAci::view_3d state_q_work_, MAMAci::const_view_2d nc, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, - MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, + MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, + MAMAci::view_2d cloud_frac_prev, MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, @@ -519,14 +512,14 @@ void call_function_dropmixnuc( icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] ekat::subview(state_q_work_loc, icol), ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] - ekat::subview(cloud_frac_new, icol), lspectype_amode, - specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, + ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, + spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, alogsig, aten, mam_idx, mam_cnst_idx, - ekat::subview(qcld, icol), // out - ekat::subview(wsub, icol), // in - ekat::subview(cloud_frac_old, icol), // in - qqcw_view, // inout + ekat::subview(qcld, icol), // out + ekat::subview(wsub, icol), // in + ekat::subview(cloud_frac_prev, icol), // in + qqcw_view, // inout ptend_q_view, ekat::subview(tendnd, icol), ekat::subview(factnum, icol), ekat::subview(ndropcol, icol), ekat::subview(ndropmix, icol), ekat::subview(nsource, icol), @@ -575,7 +568,6 @@ void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, mam_coupling::AerosolState &dry_aero_, - mam_coupling::WetAtmosphere &wet_atm_, mam_coupling::DryAtmosphere &dry_atm_, MAMAci::view_3d factnum_, MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, @@ -587,7 +579,6 @@ void call_hetfrz_compute_tendencies( mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; - mam_coupling::WetAtmosphere wet_atmosphere = wet_atm_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; @@ -714,8 +705,8 @@ void MAMAci::set_grids( Kokkos::resize(wsubice_, ncol_, nlev_); Kokkos::resize(wsig_, ncol_, nlev_); Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(cloud_frac_new_, ncol_, nlev_); - Kokkos::resize(cloud_frac_old_, ncol_, nlev_); + Kokkos::resize(cloud_frac_, ncol_, nlev_); + Kokkos::resize(cloud_frac_prev_, ncol_, nlev_); Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); Kokkos::resize(rpdel_, ncol_, nlev_); @@ -1171,10 +1162,8 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - // Negative omega means rising motion - // FIXME: Why wet_atm_ used here????? We should use dry_atm - compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, wet_atm_, - dry_atm_, top_lev_, nlev_); + compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, dry_atm_, + top_lev_, nlev_); compute_tke_using_w_sec(team_policy, tke_ /*output*/, w_sec_, nlev_); @@ -1193,9 +1182,10 @@ void MAMAci::run_impl(const double dt) { compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, dry_aero_, dry_atm_, aitken_dry_dia_, nlev_, dt); - // FIXME: Why wet_atm_ used here????? We should use dry_atm - store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, - wet_atm_, liqcldf_, liqcldf_prev_, top_lev_); + + store_liquid_cloud_fraction(team_policy, cloud_frac_, cloud_frac_prev_, + dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, + nlev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- @@ -1221,7 +1211,7 @@ void MAMAci::run_impl(const double dt) { team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, raercol_, qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atm_.p_int, dry_atm_.p_del, rpdel_, state_q_work_, dry_atm_.nc, kvh_, qcld_, wsub_, - cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, + cloud_frac_, cloud_frac_prev_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. @@ -1231,12 +1221,12 @@ void MAMAci::run_impl(const double dt) { team_policy, coltend_outp_, coltend_, nlev_); copy_mam4xx_array_to_scream( team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - // FIXME: Why wet_atm_ used here????? We should use dry_atm - call_hetfrz_compute_tendencies( - team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, factnum_, - hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, - hetfrz_depostion_nucleation_tend_, naai_hom_, naai_, diagnostic_scratch_, - nlev_, dt); + + call_hetfrz_compute_tendencies(team_policy, hetfrz_, dry_aero_, dry_atm_, + factnum_, hetfrz_immersion_nucleation_tend_, + hetfrz_contact_nucleation_tend_, + hetfrz_depostion_nucleation_tend_, naai_hom_, + naai_, diagnostic_scratch_, nlev_, dt); Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 6873ccd3397..27342467bc7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -68,8 +68,8 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_prev_; const_view_2d kvh_; - view_2d cloud_frac_new_; - view_2d cloud_frac_old_; + view_2d cloud_frac_; + view_2d cloud_frac_prev_; view_2d qcld_; view_2d tendnd_; // ptend_q_ is just ptend_q_output_ reformatted. From 4ba9252a0dd70bf92692ca16fb705bbb0bb3ce80 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 10 Apr 2024 18:00:09 -0700 Subject: [PATCH 067/142] Reaaranges args for store_liquid_cloud_fraction and add const --- .../mam/eamxx_mam_aci_process_interface.cpp | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4a7f438e89f..1eb5452219d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -295,11 +295,12 @@ void compute_nucleate_ice_tendencies( } KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac, - MAMAci::view_2d cloud_frac_prev, MAMAci::const_view_2d qc, - MAMAci::const_view_2d qi, MAMAci::const_view_2d liqcldf, - MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, - const int nlev) { + const haero::ThreadTeam &team, const MAMAci::const_view_2d qc, + const MAMAci::const_view_2d qi, const MAMAci::const_view_2d liqcldf, + const MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, + const int nlev, + // output + MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { //------------------------------------------------------------- // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value @@ -317,20 +318,22 @@ void store_liquid_cloud_fraction( } }); } -void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d cloud_frac, - MAMAci::view_2d cloud_frac_prev, - mam_coupling::DryAtmosphere &dry_atmosphere, - MAMAci::const_view_2d liqcldf, - MAMAci::const_view_2d liqcldf_prev, - const int top_lev, const int nlev) { +void store_liquid_cloud_fraction( + haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const MAMAci::const_view_2d liqcldf, + const MAMAci::const_view_2d liqcldf_prev, const int top_lev, const int nlev, + // output + MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { MAMAci::const_view_2d qc = dry_atmosphere.qc; MAMAci::const_view_2d qi = dry_atmosphere.qi; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, cloud_frac, cloud_frac_prev, qc, qi, - liqcldf, liqcldf_prev, icol, top_lev, nlev); + store_liquid_cloud_fraction(team, qc, qi, liqcldf, liqcldf_prev, icol, + top_lev, nlev, + // output + cloud_frac, cloud_frac_prev); }); } KOKKOS_INLINE_FUNCTION @@ -1183,9 +1186,10 @@ void MAMAci::run_impl(const double dt) { nidep_, nimey_, naai_hom_, naai_, dry_aero_, dry_atm_, aitken_dry_dia_, nlev_, dt); - store_liquid_cloud_fraction(team_policy, cloud_frac_, cloud_frac_prev_, - dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, - nlev_); + store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, + top_lev_, nlev_, + // output + cloud_frac_, cloud_frac_prev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- From 1c3f93076dac66f0fadba8197c5a8b2faf6d7fff Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 13:27:13 -0700 Subject: [PATCH 068/142] Fixes k loop range, adds wsubmin as input for nuc ice and re-orders in/out of funcs --- .../mam/eamxx_mam_aci_process_interface.cpp | 267 ++++++++++-------- 1 file changed, 148 insertions(+), 119 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 1eb5452219d..875f3bec883 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -35,16 +35,18 @@ void copy_scream_array_to_mam4xx( } KOKKOS_INLINE_FUNCTION -void compute_w0_and_rho(const haero::ThreadTeam &team, MAMAci::view_2d w0, - MAMAci::view_2d rho, MAMAci::const_view_2d omega, - MAMAci::const_view_2d T_mid, - MAMAci::const_view_2d p_mid, const int icol, - const int top_lev, const int nlev) { +void compute_w0_and_rho(const haero::ThreadTeam &team, + const MAMAci::const_view_2d omega, + const MAMAci::const_view_2d T_mid, + const MAMAci::const_view_2d p_mid, const int icol, + const int top_lev, const int nlev, + // output + MAMAci::view_2d w0, MAMAci::view_2d rho) { // Get physical constants using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] - static constexpr auto rair = - C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + // Gas constant for dry air [J/(kg*K) or J/Kg/K] + static constexpr auto rair = C::Rair; Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { w0(icol, kk) = 0; @@ -56,27 +58,29 @@ void compute_w0_and_rho(const haero::ThreadTeam &team, MAMAci::view_2d w0, w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); }); } -void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, - MAMAci::view_2d rho, - mam_coupling::DryAtmosphere &dry_atmosphere, - const int top_lev, const int nlev) { - // Get physical constants - using C = physics::Constants; +void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const int top_lev, const int nlev, + // output + MAMAci::view_2d w0, MAMAci::view_2d rho) { MAMAci::const_view_2d omega = dry_atmosphere.omega; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_w0_and_rho(team, w0, rho, omega, T_mid, p_mid, icol, top_lev, - nlev); + compute_w0_and_rho(team, omega, T_mid, p_mid, icol, top_lev, nlev, + // output + w0, rho); }); } KOKKOS_INLINE_FUNCTION -void compute_tke_using_w_sec(const haero::ThreadTeam &team, MAMAci::view_2d tke, - MAMAci::const_view_2d w_sec, const int icol, - const int nlev) { +void compute_tke_using_w_sec(const haero::ThreadTeam &team, + const MAMAci::const_view_2d w_sec, const int icol, + const int nlev, + // output + MAMAci::view_2d tke) { // FIXME Is this the correct boundary condition for tke at the surface? // TKE seems to be at interfaces but w_sec is at cell centers so this // descrepensy needs to be worked out. @@ -85,19 +89,23 @@ void compute_tke_using_w_sec(const haero::ThreadTeam &team, MAMAci::view_2d tke, KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); } void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d tke, MAMAci::const_view_2d w_sec, - const int nlev) { + const MAMAci::const_view_2d w_sec, const int nlev, + // output + MAMAci::view_2d tke) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_tke_using_w_sec(team, tke, w_sec, icol, nlev); + compute_tke_using_w_sec(team, w_sec, icol, nlev, + // output + tke); }); } KOKKOS_INLINE_FUNCTION void compute_subgrid_scale_velocities( - const haero::ThreadTeam &team, MAMAci::view_2d wsub, - MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, - const Real wsubmin, const int icol, const int top_lev, const int nlev) { + const haero::ThreadTeam &team, const MAMAci::const_view_2d tke, + const Real wsubmin, const int icol, const int top_lev, const int nlev, + // output + MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { // More refined computation of sub-grid vertical velocity // Set to be zero at the surface by initialization. Kokkos::parallel_for( @@ -118,14 +126,17 @@ void compute_subgrid_scale_velocities( }); } void compute_subgrid_scale_velocities( - haero::ThreadTeamPolicy team_policy, MAMAci::view_2d wsub, - MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, - const Real wsubmin, const int top_lev, const int nlev) { + haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d tke, + const Real wsubmin, const int top_lev, const int nlev, + // output + MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_subgrid_scale_velocities(team, wsub, wsubice, wsig, tke, - wsubmin, icol, top_lev, nlev); + compute_subgrid_scale_velocities(team, tke, wsubmin, icol, top_lev, + nlev, + // output + wsub, wsubice, wsig); }); } @@ -183,115 +194,124 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) { } KOKKOS_INLINE_FUNCTION void compute_subgrid_mean_updraft_velocities(const haero::ThreadTeam &team, - MAMAci::view_2d w2, - MAMAci::const_view_2d w0, - MAMAci::const_view_2d wsig, - const int icol, const int nlev) { + const MAMAci::const_view_2d w0, + const MAMAci::const_view_2d wsig, + const int icol, const int nlev, + // output + MAMAci::view_2d w2) { Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { w2(icol, kk) = subgrid_mean_updraft(w0(icol, kk), wsig(icol, kk)); }); } void compute_subgrid_mean_updraft_velocities( - haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w2, - MAMAci::const_view_2d w0, MAMAci::const_view_2d wsig, const int nlev) { + haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d w0, + const MAMAci::const_view_2d wsig, const int nlev, + // output + MAMAci::view_2d w2) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_subgrid_mean_updraft_velocities(team, w2, w0, wsig, icol, nlev); + compute_subgrid_mean_updraft_velocities(team, w0, wsig, icol, nlev, + // output + w2); }); } KOKKOS_INLINE_FUNCTION void compute_aitken_dry_diameter(const haero::ThreadTeam &team, - MAMAci::view_2d aitken_dry_dia, - MAMAci::const_view_3d dgnum, const int icol, - const int top_lev) { + const MAMAci::const_view_3d dgnum, + const int icol, const int top_lev, + const int nlev, + // output + MAMAci::view_2d aitken_dry_dia) { const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); }); } void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d aitken_dry_dia, - MAMAci::const_view_3d dgnum, - const int top_lev) { + const MAMAci::const_view_3d dgnum, + const int top_lev, const int nlev, + // output + MAMAci::view_2d aitken_dry_dia) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_aitken_dry_diameter(team, aitken_dry_dia, dgnum, icol, top_lev); + compute_aitken_dry_diameter(team, dgnum, icol, top_lev, nlev, + // output + aitken_dry_dia); }); } void compute_nucleate_ice_tendencies( const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const mam_coupling::AerosolState &dry_aerosol_state, + const MAMAci::view_2d wsubice, const MAMAci::view_2d aitken_dry_dia, + const int nlev, const double dt, + // output MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, - MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, - mam_coupling::AerosolState &dry_aerosol_state, - mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d aitken_dry_dia, - const int nlev, const double dt) { + MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai) { //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation //------------------------------------------------------------- - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - MAMAci::const_view_2d qv_dry = dry_atmosphere.qv; - MAMAci::const_view_2d cldfrac = dry_atmosphere.cldfrac; - MAMAci::const_view_2d w_updraft = dry_atmosphere.w_updraft; - using view_1d = typename KokkosTypes::template view_1d; - view_1d dummy("DummyView", nlev); - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies - // class. - haero::Surface surf{}; - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); - haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atmosphere, icol); - - // nucleation doesn't use any diagnostics, so it's okay to leave this - // alone for now - mam4::Diagnostics diags(nlev); - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - diags.dry_geometric_mean_diameter_i[aitken_idx] = - ekat::subview(aitken_dry_dia, icol); - - // These are the fields that are updated. Taking subviews means that - // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated - // in nucleate_ice.compute_tendencies. - diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); - diags.icenuc_num_immfrz = ekat::subview(niim, icol); - diags.icenuc_num_depnuc = ekat::subview(nidep, icol); - diags.icenuc_num_meydep = ekat::subview(nimey, icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t = 0; - /* - NOTE:"state_q" is a combination of subset of tracers added by - "int_mmr_field_name" and "int_nmr_field_name". Only output we care - about is "naai", "naai_hom" is never used anywhere - - Fortran code: - call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & - ! input rho, wsubice, strat_cld_frac, dgnum, & ! input naai, - naai_hom) ! output - */ - nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, - surf, progs, diags, tends); - }); + // Kokkos::parallel_for( + // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + // const int icol = team.league_rank(); + for(int icol = 0; icol < 218; ++icol) { + //--------------------------------------------------------------------- + // Set up surface, pronostics atmosphere, diagnostics, and tendencies + // classes. + //--------------------------------------------------------------------- + + // For compute_tendecies interface only, this structure is empty + haero::Surface surf{}; + + // Store interstitial and cld borne aerosols in "progrs" struture + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + + // Store atmopsheric vars (Tmid, Pmid, cloud fraction, qv, wsubmin) + haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); + + // Update the updraft velocity needed by nucleation to be "wsubice" + // in the haero_atm object + haero_atm.updraft_vel_ice_nucleation = ekat::subview(wsubice, icol); + + // All the output from this process is diagnotics; creates "diags" with + // nlev column length + mam4::Diagnostics diags(nlev); + + // Aitken mode index + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + diags.dry_geometric_mean_diameter_i[aitken_idx] = + ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated + // in nucleate_ice.compute_tendencies. + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); // not used + const mam4::AeroConfig aero_config; + const Real t = 0; // not used + nucleate_ice.compute_tendencies(aero_config, /*team,*/ t, dt, haero_atm, + surf, progs, diags, tends); + //}); + } } KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( @@ -577,9 +597,6 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], const int nlev, const double dt) { - using view_1d = typename KokkosTypes::template view_1d; - view_1d dummy("DummyView", nlev); - mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; @@ -1165,26 +1182,38 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, dry_atm_, - top_lev_, nlev_); + compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, + // output + w0_, rho_); - compute_tke_using_w_sec(team_policy, tke_ /*output*/, w_sec_, nlev_); + compute_tke_using_w_sec(team_policy, w_sec_, nlev_, + // output + tke_); Kokkos::fence(); // wait for for tke_ to be computed. - compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, - wsubmin, top_lev_, nlev_); + compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, + // output + wsub_, wsubice_, wsig_); Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); + compute_subgrid_mean_updraft_velocities(team_policy, w0_, wsig_, nlev_, + // output + w2_); - compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); + compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, + // output + aitken_dry_dia_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - // FIXME: Find out in-outs of the following call! - compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, - nidep_, nimey_, naai_hom_, naai_, dry_aero_, - dry_atm_, aitken_dry_dia_, nlev_, dt); + // Compute Ice nucleation + // NOTE: The Fortran version uses "ast" for cloud fraction which is equivalent + // to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct + compute_nucleate_ice_tendencies( + nucleate_ice_, team_policy, dry_atm_, dry_aero_, wsubice_, + aitken_dry_dia_, nlev_, dt, + // output + nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, From b4d191d8b29a47beb5a44de024d008837bb593e0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 13:51:18 -0700 Subject: [PATCH 069/142] Fixes in/outs of store_liquid_cloud_fraction --- .../mam/eamxx_mam_aci_process_interface.cpp | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 875f3bec883..d583a37d894 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -358,22 +358,26 @@ void store_liquid_cloud_fraction( } KOKKOS_INLINE_FUNCTION void compute_recipical_pseudo_density(const haero::ThreadTeam &team, - MAMAci::view_2d rpdel, - MAMAci::const_view_2d pdel, - const int icol, const int nlev) { + const MAMAci::const_view_2d pdel, + const int icol, const int nlev, + // output + MAMAci::view_2d rpdel) { // FIXME: Add an assert to ensure pdel is non-zero Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { rpdel(icol, kk) = 1 / pdel(icol, kk); }); } void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d rpdel, MAMAci::const_view_2d pdel, - const int nlev) { + const int nlev, + // output + MAMAci::view_2d rpdel) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_recipical_pseudo_density(team, rpdel, pdel, icol, nlev); + compute_recipical_pseudo_density(team, pdel, icol, nlev, + // output + rpdel); }); } @@ -1237,8 +1241,16 @@ void MAMAci::run_impl(const double dt) { rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, dry_atm_.p_del, nlev_); + compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, + // output + rpdel_); + Kokkos::fence(); // wait for rpdel_ to be computed. + /*call dropmixnuc(lchnk, ncol, deltatin, temperature, pmid, pint, pdel, rpdel, + zm, & ! in + state_q, nc, kvh, wsub, lcldn, lcldo, & ! in + qqcw, & ! inout + ptend, nctend_mixnuc, factnum) !out*/ call_function_dropmixnuc( team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, raercol_, From 449af2c98652c0cfcbb374f940db29697551c492 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 14:25:54 -0700 Subject: [PATCH 070/142] Removes atm vars from the dropmixnuc arg list --- .../mam/eamxx_mam_aci_process_interface.cpp | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index d583a37d894..475f131cd2e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -390,11 +390,9 @@ void call_function_dropmixnuc( MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], - MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, MAMAci::view_3d state_q_work_, - MAMAci::const_view_2d nc, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, - MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d rpdel, + MAMAci::view_3d state_q_work_, MAMAci::const_view_2d kvh, + MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev, MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, @@ -405,6 +403,9 @@ void call_function_dropmixnuc( MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; MAMAci::const_view_2d zm = dry_atmosphere.z_mid; + MAMAci::const_view_2d pdel = dry_atmosphere.p_del; + MAMAci::const_view_2d p_int = dry_atmosphere.p_int; + MAMAci::const_view_2d nc = dry_atmosphere.nc; MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; @@ -1250,14 +1251,14 @@ void MAMAci::run_impl(const double dt) { zm, & ! in state_q, nc, kvh, wsub, lcldn, lcldo, & ! in qqcw, & ! inout - ptend, nctend_mixnuc, factnum) !out*/ - - call_function_dropmixnuc( - team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, raercol_, - qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atm_.p_int, - dry_atm_.p_del, rpdel_, state_q_work_, dry_atm_.nc, kvh_, qcld_, wsub_, - cloud_frac_, cloud_frac_prev_, tendnd_, factnum_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + ptend, nctend_mixnuc, factnum) !out*/ + + call_function_dropmixnuc(team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, + raercol_, qqcw_fld_work_, ptend_q_, coltend_, + coltend_cw_, rpdel_, state_q_work_, kvh_, qcld_, + wsub_, cloud_frac_, cloud_frac_prev_, tendnd_, + factnum_, ndropcol_, ndropmix_, nsource_, wtke_, + ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( From e2c12786df6d94570a6a1d3be630bef709980883 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 15:47:35 -0700 Subject: [PATCH 071/142] Moves some i/o vars in dropmixnuc call, not done yet --- .../mam/eamxx_mam_aci_process_interface.cpp | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 475f131cd2e..f6b14f71d70 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -382,22 +382,26 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, } void call_function_dropmixnuc( - haero::ThreadTeamPolicy team_policy, - mam_coupling::DryAtmosphere &dry_atmosphere, - const mam_coupling::AerosolState &dry_aerosol_state, const Real dt, + haero::ThreadTeamPolicy team_policy, const Real dt, + mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d rpdel, + MAMAci::const_view_2d kvh, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, + MAMAci::view_2d cloud_frac_prev, + const mam_coupling::AerosolState &dry_aerosol_state, + + // output + MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_3d factnum, + MAMAci::view_2d tendnd, MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], + + // work arrays MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d rpdel, - MAMAci::view_3d state_q_work_, MAMAci::const_view_2d kvh, - MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, - MAMAci::view_2d cloud_frac_prev, MAMAci::view_2d tendnd, - MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, - MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - MAMAci::view_3d nact, MAMAci::view_3d mact, - MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { + MAMAci::view_3d state_q_work_, MAMAci::view_2d qcld, + MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, + MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, + MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], + const int nlev) { // FIXME: why can't we use MAMAci::dropmix_scratch_ above MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; @@ -1253,12 +1257,14 @@ void MAMAci::run_impl(const double dt) { qqcw, & ! inout ptend, nctend_mixnuc, factnum) !out*/ - call_function_dropmixnuc(team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, - raercol_, qqcw_fld_work_, ptend_q_, coltend_, - coltend_cw_, rpdel_, state_q_work_, kvh_, qcld_, - wsub_, cloud_frac_, cloud_frac_prev_, tendnd_, - factnum_, ndropcol_, ndropmix_, nsource_, wtke_, - ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + call_function_dropmixnuc( + team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, cloud_frac_, + cloud_frac_prev_, dry_aero_, + // output + qqcw_fld_work_, ptend_q_, factnum_, tendnd_, coltend_, coltend_cw_, + // work arrays + raercol_cw_, raercol_, state_q_work_, qcld_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( From ab242612ac8ca3392c976c1242dfb86a4d7a5f5e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 16:01:41 -0700 Subject: [PATCH 072/142] Moves ndrop init functions outside of kokkos loop --- .../mam/eamxx_mam_aci_process_interface.cpp | 77 ++++++++++--------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index f6b14f71d70..637469d0678 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -386,7 +386,7 @@ void call_function_dropmixnuc( mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d rpdel, MAMAci::const_view_2d kvh, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev, - const mam_coupling::AerosolState &dry_aerosol_state, + const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, // output MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], @@ -400,8 +400,7 @@ void call_function_dropmixnuc( MAMAci::view_3d state_q_work_, MAMAci::view_2d qcld, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, - MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], - const int nlev) { + MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15]) { // FIXME: why can't we use MAMAci::dropmix_scratch_ above MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; @@ -434,48 +433,56 @@ void call_function_dropmixnuc( MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; - for(int i = 0; i < mam4::ndrop::pver; ++i) - for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; - for(int i = 0; i < mam4::ndrop::pver; ++i) - for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + loc_raercol_cw[i][j] = raercol_cw[i][j]; + loc_raercol[i][j] = raercol[i][j]; + } + } + + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + loc_coltend[i] = coltend[i]; loc_coltend_cw[i] = coltend_cw[i]; + } + + for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; MAMAci::view_2d qqcw_fld_work_loc[25]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; MAMAci::view_3d state_q_work_loc = state_q_work_; + //--------------------------------------------------------------------------- + // ## Initialize the ndrop class. + //--------------------------------------------------------------------------- + const int ntot_amode = mam_coupling::num_aero_modes(); + const int maxd_aspectype = mam4::ndrop::maxd_aspectype; + const int nspec_max = mam4::ndrop::nspec_max; + int nspec_amode[ntot_amode] = {}; + int lspectype_amode[maxd_aspectype][ntot_amode] = {}; + int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; + Real specdens_amode[maxd_aspectype] = {}; + Real spechygro[maxd_aspectype] = {}; + int numptr_amode[ntot_amode] = {}; + int mam_idx[ntot_amode][nspec_max] = {}; + int mam_cnst_idx[ntot_amode][nspec_max] = {}; + Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, + num2vol_ratio_min_nmodes[ntot_amode] = {}, + num2vol_ratio_max_nmodes[ntot_amode] = {}; + Real aten = 0; + mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, + numptr_amode, specdens_amode, spechygro, + mam_idx, mam_cnst_idx); + mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, + num2vol_ratio_max_nmodes); + //--------------------------------------------------------------------------- + // ## END (Initialize the ndrop class.) + //--------------------------------------------------------------------------- Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // Initialize the ndrop class. - const int ntot_amode = mam_coupling::num_aero_modes(); - const int maxd_aspectype = mam4::ndrop::maxd_aspectype; - const int nspec_max = mam4::ndrop::nspec_max; - int nspec_amode[ntot_amode] = {}; - int lspectype_amode[maxd_aspectype][ntot_amode] = {}; - int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; - Real specdens_amode[maxd_aspectype] = {}; - Real spechygro[maxd_aspectype] = {}; - int numptr_amode[ntot_amode] = {}; - int mam_idx[ntot_amode][nspec_max] = {}; - int mam_cnst_idx[ntot_amode][nspec_max] = {}; - Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, - num2vol_ratio_min_nmodes[ntot_amode] = {}, - num2vol_ratio_max_nmodes[ntot_amode] = {}; - Real aten = 0; - mam4::ndrop::get_e3sm_parameters( - nspec_amode, lspectype_amode, lmassptr_amode, numptr_amode, - specdens_amode, spechygro, mam_idx, mam_cnst_idx); - mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, - num2vol_ratio_min_nmodes, // voltonumbhi_amode - num2vol_ratio_max_nmodes); // voltonumblo_amode - mam4::ndrop::View1D raercol_cw_view[mam4::ndrop::pver][2]; mam4::ndrop::View1D raercol_view[mam4::ndrop::pver][2]; for(int i = 0; i < mam4::ndrop::pver; ++i) { @@ -1259,12 +1266,12 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc( team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, cloud_frac_, - cloud_frac_prev_, dry_aero_, + cloud_frac_prev_, dry_aero_, nlev_, // output qqcw_fld_work_, ptend_q_, factnum_, tendnd_, coltend_, coltend_cw_, // work arrays raercol_cw_, raercol_, state_q_work_, qcld_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( From 8c9685533a0c74b5783942fb341f21b1a430abe0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 18:16:53 -0700 Subject: [PATCH 073/142] Moves some variables into output and moves some code around in dropmixnuc --- .../mam/eamxx_mam_aci_process_interface.cpp | 105 ++++++++++-------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 637469d0678..00addc2c5ea 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -383,24 +383,25 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, const Real dt, - mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d rpdel, - MAMAci::const_view_2d kvh, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, - MAMAci::view_2d cloud_frac_prev, + mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, + const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, + const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, // output MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_3d factnum, MAMAci::view_2d tendnd, MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, + MAMAci::view_2d ndropcol, // work arrays MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_3d state_q_work_, MAMAci::view_2d qcld, - MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, - MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, - MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15]) { + MAMAci::view_3d state_q_work_, MAMAci::view_2d ndropmix, + MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, + MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_2d dropmixnuc_scratch_mem[15]) { // FIXME: why can't we use MAMAci::dropmix_scratch_ above MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; @@ -410,22 +411,10 @@ void call_function_dropmixnuc( MAMAci::const_view_2d p_int = dry_atmosphere.p_int; MAMAci::const_view_2d nc = dry_atmosphere.nc; - MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; - MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; - MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; - MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; - MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; - MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; - MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; - MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; - MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; - MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; - MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; - MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; - MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; - MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; - MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; - + //---------------------------------------------------------------------- + // ## Declare local variables for class variables + //(FIXME: GPU hack, revisit this) + //---------------------------------------------------------------------- MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; MAMAci::view_2d loc_qqcw[mam4::ndrop::ncnst_tot]; @@ -452,6 +441,27 @@ void call_function_dropmixnuc( qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; MAMAci::view_3d state_q_work_loc = state_q_work_; + + //---------------------------------------------------------------------- + // ## Assign scratch memory for work variables + //---------------------------------------------------------------------- + + MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; + MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; + MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; + MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; + MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; + MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; + MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; + MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; + MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; + MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; + MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; + MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; + MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; + MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; + MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; + //--------------------------------------------------------------------------- // ## Initialize the ndrop class. //--------------------------------------------------------------------------- @@ -476,7 +486,6 @@ void call_function_dropmixnuc( mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes); //--------------------------------------------------------------------------- - // ## END (Initialize the ndrop class.) //--------------------------------------------------------------------------- Kokkos::parallel_for( @@ -506,34 +515,31 @@ void call_function_dropmixnuc( coltend_cw_view[i] = ekat::subview(loc_coltend_cw[i], icol); } - /* - NOTE: "deltain" is the model time step. "state_q" is a combination of - tracers fields with "int_mmr_field_name" and "int_nmr_field_name". - "z_mid" is computed. "qqcw" is the combination of cld_mmr_field_name and - cld_nmr_field_name. The output "ptend" will have tendencies for - interstitial and cloud borne aerosols. - - Fortan code: - call dropmixnuc(lchnk, ncol, deltatin, T_mid, p_mid, p_int, p_del, - rpdel, z_mid, & ! in state_q, nc, kvh, wsub, lcldn, lcldo, & ! in - qqcw, & ! inout - ptend, nctend_mixnuc, factnum) !out - */ + // To construct state_q, we need fields from Prognostics (aerosols) + // and Atmosphere (water species such as qv, qc etc.) + // get prognostics mam4::Prognostics progs_at_col = aerosols_for_column(dry_aerosol_state, icol); + + // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); + // Construct state_q (interstitial) and qqcw (cloud borne) arrays for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = - {}; // use pcnst here - Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a gri cell (col,lev) mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); + // get qqcw at a gri cell (col,lev) mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); + + // create colum views of state_q and qqcw for(int icnst = 15; icnst < mam4::aero_model::pcnst; ++icnst) { state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; // FIXME: ensure that indices are @@ -1264,14 +1270,15 @@ void MAMAci::run_impl(const double dt) { qqcw, & ! inout ptend, nctend_mixnuc, factnum) !out*/ - call_function_dropmixnuc( - team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, cloud_frac_, - cloud_frac_prev_, dry_aero_, nlev_, - // output - qqcw_fld_work_, ptend_q_, factnum_, tendnd_, coltend_, coltend_cw_, - // work arrays - raercol_cw_, raercol_, state_q_work_, qcld_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_); + call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, + cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, + // output + qqcw_fld_work_, ptend_q_, factnum_, tendnd_, + coltend_, coltend_cw_, qcld_, ndropcol_, + // work arrays + raercol_cw_, raercol_, state_q_work_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, + dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( From 5c221ab09f0e746f3a98547273c6f0c164d515ca Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 18:28:39 -0700 Subject: [PATCH 074/142] Moves some more fields to output --- .../mam/eamxx_mam_aci_process_interface.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 00addc2c5ea..bb419847e45 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -393,17 +393,17 @@ void call_function_dropmixnuc( MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_3d factnum, MAMAci::view_2d tendnd, MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, - MAMAci::view_2d ndropcol, + MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, + MAMAci::view_2d wtke, MAMAci::view_3d ccn, // work arrays MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], - MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_3d state_q_work_, MAMAci::view_2d ndropmix, - MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, + MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15]) { - // FIXME: why can't we use MAMAci::dropmix_scratch_ above + // FIXME: why can't we use MAMAci::dropmix_scratch_ above instead of 15? + // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; MAMAci::const_view_2d zm = dry_atmosphere.z_mid; @@ -440,7 +440,7 @@ void call_function_dropmixnuc( for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; - MAMAci::view_3d state_q_work_loc = state_q_work_; + MAMAci::view_3d state_q_work_loc = state_q_work; //---------------------------------------------------------------------- // ## Assign scratch memory for work variables @@ -1274,10 +1274,10 @@ void MAMAci::run_impl(const double dt) { cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output qqcw_fld_work_, ptend_q_, factnum_, tendnd_, - coltend_, coltend_cw_, qcld_, ndropcol_, + coltend_, coltend_cw_, qcld_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, // work arrays - raercol_cw_, raercol_, state_q_work_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, + raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. From 9235c84948e5b6f484b55ee73fa574fcd9a448c5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Apr 2024 09:13:42 -0700 Subject: [PATCH 075/142] Adds wsubmin namelist and also a lot of debugging cout stamenents for Fortran verification --- .../cime_config/namelist_defaults_scream.xml | 4 + .../mam/eamxx_mam_aci_process_interface.cpp | 139 +++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 5 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 9 ++ .../tests/single-process/mam/aci/input.yaml | 15 +- 5 files changed, 129 insertions(+), 43 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 9a33879b01d..8aa4c0ea6f7 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -234,6 +234,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.1 + + + 0.001 + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index bb419847e45..158f88dcd9d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -388,15 +388,26 @@ void call_function_dropmixnuc( const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, - // output - MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_3d factnum, - MAMAci::view_2d tendnd, MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + // ## outputs ## + // qqcw_fld_work should be directly assigned to the cloud borne aerosols + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + + // ptend_q are the tendencies to the interstitial aerosols + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + + // factnum is used by the hetrozenous freezing + MAMAci::view_3d factnum, + + // tendnd is used by microphysics scheme (e.g. P3) + MAMAci::view_2d tendnd, + + // Following outputs are all diagnostics + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - // work arrays + // ## work arrays ## MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, @@ -471,11 +482,12 @@ void call_function_dropmixnuc( int nspec_amode[ntot_amode] = {}; int lspectype_amode[maxd_aspectype][ntot_amode] = {}; int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; - Real specdens_amode[maxd_aspectype] = {}; - Real spechygro[maxd_aspectype] = {}; int numptr_amode[ntot_amode] = {}; int mam_idx[ntot_amode][nspec_max] = {}; int mam_cnst_idx[ntot_amode][nspec_max] = {}; + + Real specdens_amode[maxd_aspectype] = {}; + Real spechygro[maxd_aspectype] = {}; Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, num2vol_ratio_min_nmodes[ntot_amode] = {}, num2vol_ratio_max_nmodes[ntot_amode] = {}; @@ -552,11 +564,9 @@ void call_function_dropmixnuc( team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), ekat::subview(pdel, icol), ekat::subview(rpdel, icol), - ekat::subview( - zm, - icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - ekat::subview(state_q_work_loc, icol), ekat::subview(nc, icol), - ekat::subview(kvh, icol), // kvh[kk+1] + // in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] + ekat::subview(zm, icol), ekat::subview(state_q_work_loc, icol), + ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, @@ -612,23 +622,24 @@ void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, - mam_coupling::AerosolState &dry_aero_, - mam_coupling::DryAtmosphere &dry_atm_, MAMAci::view_3d factnum_, + mam_coupling::DryAtmosphere &dry_atm_, + mam_coupling::AerosolState &dry_aero_, MAMAci::view_3d factnum_, + const double dt, const int nlev, + // output MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, - MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, - MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], const int nlev, - const double dt) { + MAMAci::view_2d hetfrz_depostion_nucleation_tend, + MAMAci::view_2d diagnostic_scratch_[]) { mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; + MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // for (int icol=0; icol<1; icol++){ // Set up an atmosphere, surface, diagnostics, pronostics and // tendencies class. @@ -641,7 +652,6 @@ void call_hetfrz_compute_tendencies( const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - // BALLI mam4::Diagnostics diags(nlev); diags.activation_fraction[accum_idx] = @@ -712,19 +722,19 @@ void call_hetfrz_compute_tendencies( // values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; - const Real t = 0; //, dt = 0; + const Real t = 0; hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, diags, tends); }); - //} } } // namespace -// FIXME: The following variables are namelist variables -const Real wsubmin = 1; - MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) - : AtmosphereProcess(comm, params) {} + : AtmosphereProcess(comm, params) { + // Asserts for the runtime or namelist options + EKAT_REQUIRE_MSG(m_params.isParameter("wsubmin"), + "ERROR: wsubmin is missing from mam_optics parameter list."); +} // ================================================================ // SET_GRIDS @@ -1028,6 +1038,12 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { // INITIALIZE_IMPL // ================================================================ void MAMAci::initialize_impl(const RunType run_type) { + // ------------------------------------------------------------------------ + // ## Runtime options + // ------------------------------------------------------------------------ + + ACIRuntime_.wsubmin = m_params.get("wsubmin"); + // ------------------------------------------------------------------------ // Input fields read in from IC file, namelist or other processes // ------------------------------------------------------------------------ @@ -1204,30 +1220,79 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + const int kb = 66; compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); + // std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; + // std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; + std::cout << "wsec:" << w_sec_(0, kb) << std::endl; + std::cout << "w0:" << w0_(0, kb) << std::endl; + std::cout << " rho: " << rho_(0, kb) << std::endl; compute_tke_using_w_sec(team_policy, w_sec_, nlev_, // output tke_); + const Real tke_b[73] = { + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00062854116332895, 0.00067291680645447, 0.00101664413803513, + 0.00232786652593969, 0.00569557505990274, 0.01000238209664803, + 0.01519680083369616, 0.02170550249412443, 0.02401690416836010, + 0.02366486534737443, 0.02153664341694223, 0.01578233182683715, + 0.01145733753135718, 0.01010923067444657, 0.01243858801725962, + 0.02323111869706078, 0.04796154961104696, 0.08572647178140329, + 0.12086080769994784, 0.14941194987050846, 0.15643944255461628, + 0.14719266313675605}; + for(int kk = 0; kk < 73; ++kk) tke_(0, kk) = tke_b[kk]; + + std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. - + const Real &wsubmin = ACIRuntime_.wsubmin; // runtime variable compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); + std::cout << "WSUB:" << wsub_(0, kb) << std::endl; + std::cout << "WICE:" << wsubice_(0, kb) << std::endl; + std::cout << "WSIG:" << wsig_(0, kb) << std::endl; + Kokkos::fence(); // wait for wsig_ to be computed. compute_subgrid_mean_updraft_velocities(team_policy, w0_, wsig_, nlev_, // output w2_); + std::cout << "W2_:" << w2_(0, kb) << std::endl; + /* wo 9.784350381198358E-003 + rho: 1.07565908612486 + TKE: 2.323111869706078E-002 4.796154961104696E-002 + wsub: 0.154048334739574 + wsubi: 0.200000000000000 + wsig: 0.154048334739574 + w2: 6.571878680006606E-002*/ + compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is equivalent // to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct @@ -1264,11 +1329,6 @@ void MAMAci::run_impl(const double dt) { rpdel_); Kokkos::fence(); // wait for rpdel_ to be computed. - /*call dropmixnuc(lchnk, ncol, deltatin, temperature, pmid, pint, pdel, rpdel, - zm, & ! in - state_q, nc, kvh, wsub, lcldn, lcldo, & ! in - qqcw, & ! inout - ptend, nctend_mixnuc, factnum) !out*/ call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, @@ -1288,12 +1348,17 @@ void MAMAci::run_impl(const double dt) { copy_mam4xx_array_to_scream( team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - call_hetfrz_compute_tendencies(team_policy, hetfrz_, dry_aero_, dry_atm_, - factnum_, hetfrz_immersion_nucleation_tend_, - hetfrz_contact_nucleation_tend_, - hetfrz_depostion_nucleation_tend_, naai_hom_, - naai_, diagnostic_scratch_, nlev_, dt); + /* call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, + & ! in qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in + frzimm, frzcnt, frzdep) */ + call_hetfrz_compute_tendencies( + team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, + // output + hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, + hetfrz_depostion_nucleation_tend_, + // work arrays + diagnostic_scratch_); Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 27342467bc7..3159fa3f0be 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -32,9 +32,12 @@ class MAMAci final : public scream::AtmosphereProcess { using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; - // FIXME:B: Should the following variables be public? They are like that in + // FIXME: Should the following variables be public? They are like that in // micriphysics and optics codes + // ACI runtime ( or namelist) options + mam_coupling::MAM4ACIRuntime ACIRuntime_; + // rho is air density [kg/m3] view_2d rho_; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 6b5cfbd186d..3c01f469092 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -252,6 +252,15 @@ const char* gas_mmr_field_name(const int gas) { return const_cast(gas_species_name(gas)); } + // This struct stores runtime options for MAM4xx-ACI + struct MAM4ACIRuntime { + MAM4ACIRuntime() = default; + // Runtime options for MAM4xx aerosol cloud interactions code + + Real wsubmin; //Minimum diagnostic sub-grid vertical velocity + }; + + // This type stores multi-column views related specifically to the wet // atmospheric state used by EAMxx. struct WetAtmosphere { diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index dc1b779875a..4941b2d259b 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -10,6 +10,8 @@ time_stepping: atmosphere_processes: atm_procs_list: [mam4_aci] + mam4_aci: + wsubmin: 0.001 grids_manager: Type: Mesh Free @@ -30,11 +32,14 @@ initial_conditions: #we should get the following variables from other processes pbl_height : 1.0 - dgnum: 1e-3 - eddy_diff_heat: 1e-3 - cldfrac_liq: 1e-3 - cldfrac_liq_prev: 0.1 - w_variance: 1e-3 + p_mid: 86389.8729712384 + omega: -0.103206160508178 + T_mid: 279.796493114264 + dgnum: 1.326982247461267E-007 + eddy_diff_heat: 2.82379273906116 + cldfrac_liq: 0.999966505294370 + cldfrac_liq_prev: 0.999999418697814 + w_variance: 1.548741246470719E-002 # The parameters for I/O control Scorpio: From f179ffbbeca5acd0ceb4e1e9a7ca10e95db83ccb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Apr 2024 09:22:06 -0700 Subject: [PATCH 076/142] Removes subgrid_mean_updraft velocities functions as they are not used --- .../mam/eamxx_mam_aci_process_interface.cpp | 84 +------------------ 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 158f88dcd9d..45d5d9b0bee 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -140,83 +140,6 @@ void compute_subgrid_scale_velocities( }); } -KOKKOS_INLINE_FUNCTION -Real subgrid_mean_updraft(const Real w0, const Real wsig) { - /* --------------------------------------------------------------------------------- - Purpose: Calculate the mean updraft velocity inside a GCM grid assuming the - vertical velocity distribution is Gaussian and peaks at the - GCM resolved large-scale vertical velocity. - When icenul_wsub_scheme = 2, the model uses the mean updraft - velocity as the characteristic updraft velocity to calculate the ice - nucleation rate. Author: Kai Zhang (kai.zhang@pnnl.gov) Last Modified: Oct, - 2015 - -------------------------------------------------------------------------------- -*/ - - // interface - - // in :: wsig standard deviation (m/s) - // in :: w0 large scale vertical velocity (m/s) - // out:: mean updraft velocity(m/s) -> characteristic w* - - // FIXME should nbin be a user parameter? - const int nbin = 50; - - using C = physics::Constants; - constexpr Real pi = C::Pi; - Real zz[nbin], wa = 0, ww = 0; - int kp = 0; - - const Real sigma = haero::max(0.001, wsig); - const Real wlarge = w0; - - const Real xx = 6.0 * sigma / nbin; - - for(int ibin = 0; ibin < nbin; ++ibin) { - Real yy = wlarge - 3.0 * sigma + 0.5 * xx; - yy += (ibin - 1) * xx; - // wbar = integrator < w * f(w) * dw > - zz[ibin] = - yy * - haero::exp(-1.0 * haero::square(yy - wlarge) / (2 * sigma * sigma)) / - (sigma * haero::sqrt(2 * pi)) * xx; - } - for(int ibin = 0; ibin < nbin; ++ibin) { - if(zz[ibin] > 0) ++kp, wa += zz[ibin]; - } - if(kp) { - // wbar = integrator < w * f(w) * dw > - ww = wa; - } else { - ww = 0.001; - } - return ww; -} -KOKKOS_INLINE_FUNCTION -void compute_subgrid_mean_updraft_velocities(const haero::ThreadTeam &team, - const MAMAci::const_view_2d w0, - const MAMAci::const_view_2d wsig, - const int icol, const int nlev, - // output - MAMAci::view_2d w2) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - w2(icol, kk) = subgrid_mean_updraft(w0(icol, kk), wsig(icol, kk)); - }); -} -void compute_subgrid_mean_updraft_velocities( - haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d w0, - const MAMAci::const_view_2d wsig, const int nlev, - // output - MAMAci::view_2d w2) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_subgrid_mean_updraft_velocities(team, w0, wsig, icol, nlev, - // output - w2); - }); -} KOKKOS_INLINE_FUNCTION void compute_aitken_dry_diameter(const haero::ThreadTeam &team, const MAMAci::const_view_3d dgnum, @@ -449,7 +372,7 @@ void call_function_dropmixnuc( MAMAci::view_2d qqcw_fld_work_loc[25]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) - qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; + qqcw_fld_work_loc[i] = qqcw_fld_work[i]; MAMAci::view_3d state_q_work_loc = state_q_work; @@ -1274,11 +1197,6 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w0_, wsig_, nlev_, - // output - w2_); - - std::cout << "W2_:" << w2_(0, kb) << std::endl; /* wo 9.784350381198358E-003 rho: 1.07565908612486 TKE: 2.323111869706078E-002 4.796154961104696E-002 From dbbd5a549a3a5ae867b1aeb5c40813ff8417ad03 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Apr 2024 07:07:35 -0700 Subject: [PATCH 077/142] One grid cell validated code with all debug statements --- .../mam/eamxx_mam_aci_process_interface.cpp | 198 ++++++++++++------ .../eamxx/src/physics/mam/mam_coupling.hpp | 20 +- .../single-process/mam/aci/CMakeLists.txt | 4 +- .../tests/single-process/mam/aci/input.yaml | 13 +- 4 files changed, 154 insertions(+), 81 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 45d5d9b0bee..1d47fdcc559 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -184,7 +184,9 @@ void compute_nucleate_ice_tendencies( // Kokkos::parallel_for( // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { // const int icol = team.league_rank(); - for(int icol = 0; icol < 218; ++icol) { + for(int icol = 0; icol < 5; ++icol) { + //std::cout<<""<("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -774,7 +779,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -1133,84 +1138,69 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { + //for (int kk=0; kk<72; ++kk){ + //std::cout << "num_cb_accum_top:"<::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + //KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); - const int kb = 66; + const int kb = 62; compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); - // std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; - // std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; - std::cout << "wsec:" << w_sec_(0, kb) << std::endl; - std::cout << "w0:" << w0_(0, kb) << std::endl; - std::cout << " rho: " << rho_(0, kb) << std::endl; + //std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; + //std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; + //std::cout << "wsec:" << w_sec_(0, kb) << std::endl; + //std::cout << "w0:" << w0_(0, kb) << std::endl; + //std::cout << " rho: " << rho_(0, kb) << std::endl; compute_tke_using_w_sec(team_policy, w_sec_, nlev_, // output tke_); - const Real tke_b[73] = { - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00062854116332895, 0.00067291680645447, 0.00101664413803513, - 0.00232786652593969, 0.00569557505990274, 0.01000238209664803, - 0.01519680083369616, 0.02170550249412443, 0.02401690416836010, - 0.02366486534737443, 0.02153664341694223, 0.01578233182683715, - 0.01145733753135718, 0.01010923067444657, 0.01243858801725962, - 0.02323111869706078, 0.04796154961104696, 0.08572647178140329, - 0.12086080769994784, 0.14941194987050846, 0.15643944255461628, - 0.14719266313675605}; - for(int kk = 0; kk < 73; ++kk) tke_(0, kk) = tke_b[kk]; - - std::cout << "TKE:" << tke_(0, kb) << std::endl; + //std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. const Real &wsubmin = ACIRuntime_.wsubmin; // runtime variable compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); - std::cout << "WSUB:" << wsub_(0, kb) << std::endl; - std::cout << "WICE:" << wsubice_(0, kb) << std::endl; - std::cout << "WSIG:" << wsig_(0, kb) << std::endl; + //std::cout << "WSUB:" << wsub_(0, kb) << std::endl; + //std::cout << "WICE:" << wsubice_(0, kb) << std::endl; + //std::cout << "WSIG:" << wsig_(0, kb) << std::endl; Kokkos::fence(); // wait for wsig_ to be computed. - /* wo 9.784350381198358E-003 - rho: 1.07565908612486 - TKE: 2.323111869706078E-002 4.796154961104696E-002 - wsub: 0.154048334739574 - wsubi: 0.200000000000000 - wsig: 0.154048334739574 - w2: 6.571878680006606E-002*/ + /* wo 3.343177351577457E-004 + rho: 1.28186433384113 + TKE: 1.666465903934813E-002 6.097664104873541E-002 + wsub: 0.160873967324407 + wsubi: 0.200000000000000 + wsig: 0.160873967324407 + dgnum_ait: 4.362354500358337E-008 + naai: 11464.5957222634 + naai_hom: 29838.6879763933 + */ compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); + const Real dgnum_ait_e3sm[72] = { 0.20877713336487552E-007, 0.21782230353342090E-007, 0.21688324003865861E-007, 0.21112855042342451E-007, 0.19162058462939536E-007, 0.18102979880838476E-007, 0.17906980715477606E-007, 0.20271254074583327E-007, 0.22698983422181942E-007, 0.24134835117044986E-007, 0.25498156808001372E-007, 0.29796738799905547E-007, 0.35822987394021908E-007, 0.41170963764365215E-007, 0.44892726528330642E-007, 0.47217231342203107E-007, 0.48928661807108766E-007, 0.50170939816128735E-007, 0.51078750853732200E-007, 0.52247333465736065E-007, 0.53190758580174931E-007, 0.53576491941850044E-007, 0.53915614473890715E-007, 0.54510964775236826E-007, 0.55643231691556703E-007, 0.57057811112589899E-007, 0.58177383586181116E-007, 0.58209849180850108E-007, 0.57976751598840998E-007, 0.52000000000000002E-007, 0.50728746567226150E-007, 0.49119902704480870E-007, 0.48212162162050883E-007, 0.49227715213506454E-007, 0.46876827233752246E-007, 0.45360603896257791E-007, 0.49986783979004747E-007, 0.51186879246229022E-007, 0.50009353247048599E-007, 0.48250264542204811E-007, 0.47560278748093609E-007, 0.48298089720730957E-007, 0.49095935613468768E-007, 0.49493024126912931E-007, 0.50250797590476007E-007, 0.51949267668322422E-007, 0.53778727208416418E-007, 0.53563593301099588E-007, 0.51218136771199298E-007, 0.43171429694325200E-007, 0.39019610039033895E-007, 0.36175109143257051E-007, 0.42731638777892750E-007, 0.38060728507221777E-007, 0.44046323901481340E-007, 0.39216732751330010E-007, 0.34842233953609988E-007, 0.34068804733226066E-007, 0.30636043694263528E-007, 0.28302341686131413E-007, 0.33023014309036320E-007, 0.34745748365385196E-007, 0.43623545003583371E-007, 0.48206451795644064E-007, 0.49854490325455530E-007, 0.50346335647724146E-007, 0.50661560988561763E-007, 0.50986261962838767E-007, 0.51256955985111086E-007, 0.51482578449096488E-007, 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < 218; ++icol){ + for(int kk = 0; kk < 72; ++kk){ + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + }} Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; + //std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is equivalent // to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct @@ -1225,6 +1215,9 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); + //std::cout << "naai_hom_:" << naai_hom_(0, kb) << std::endl; + //std::cout << "naai_:" << naai_(0, kb) << std::endl; + // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- // Save cloud borne aerosols to be used in the heterozenous @@ -1245,6 +1238,7 @@ void MAMAci::run_impl(const double dt) { compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); + //std::cout << "rpdel_:" << rpdel_(0, kb) << std::endl; Kokkos::fence(); // wait for rpdel_ to be computed. @@ -1258,6 +1252,17 @@ void MAMAci::run_impl(const double dt) { raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. + + + //std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, kb)<<" : "<( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1266,10 +1271,6 @@ void MAMAci::run_impl(const double dt) { copy_mam4xx_array_to_scream( team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - /* call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, - & ! in qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in - frzimm, frzcnt, frzdep) */ - call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, // output @@ -1277,6 +1278,10 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); + std::cout << "hetfrz_immersion_nucleation_tend_:" << hetfrz_immersion_nucleation_tend_(0, kb)<< std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" << hetfrz_contact_nucleation_tend_(0, kb)<< std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" << hetfrz_depostion_nucleation_tend_(0, kb)<< std::endl; + Kokkos::fence(); // wait before returning to calling function } @@ -1285,3 +1290,78 @@ void MAMAci::finalize_impl() { } } // namespace scream + //tendnd: -839.404918218856 7289315.36108503 7792958.31201634 1.666666666666667E-003 + //factnum: 0.998556176544263 0.861689826773470 0.999999999974140 0.000000000000000E+000 + /*nctend_mixnuc: -839.404918218856 + O3_tend: 0.000000000000000E+000 + H2O2_tend: 0.000000000000000E+000 + H2SO4_tend: 0.000000000000000E+000 + SO2_tend: 0.000000000000000E+000 + DMS_tend: 0.000000000000000E+000 + SOAG_tend: 0.000000000000000E+000 + so4_a1_tend: -1.384310943569190E-014 + pom_a1_tend: -2.950799002682271E-015 + soa_a1_tend: -2.283869371271150E-014 + bc_a1_tend: -2.202672700280516E-016 + dst_a1_tend: -2.194060414083922E-016 + ncl_a1_tend: -6.981124745267083E-017 + mom_a1_tend: -5.662690535048673E-018 + num_a1_tend: -1626.38730532537 + so4_a2_tend: -2.548707408341951E-017 + soa_a2_tend: -1.929990276139139E-018 + ncl_a2_tend: -6.855087738119723E-019 + mom_a2_tend: -4.571691604185304E-020 + num_a2_tend: -111.890330245159 + dst_a3_tend: -7.609611333715698E-015 + ncl_a3_tend: -1.256350252705383E-015 + so4_a3_tend: -1.320988655880598E-016 + bc_a3_tend: -1.099586316024402E-018 + pom_a3_tend: -3.309939677824050E-018 + soa_a3_tend: -2.767668335981830E-017 + mom_a3_tend: -2.254911604324925E-019 + num_a3_tend: -0.406788905791186 + pom_a4_tend: -2.375070578317096E-017 + bc_a4_tend: 4.082479713528180E-017 + mom_a4_tend: -4.901356204764327E-023 + num_a4_tend: -34.4232851017138 + so4_c1: 1322558.40616450 + pom_c1: 1.233997159684093E-011 + soa_c1: 2.393397505793018E-012 + bc_c1: 1.872520079842212E-011 + dst_c1: 1.922157644491776E-013 + ncl_c1: 2.530385880698310E-013 + mom_c1: 8.315014074126540E-014 + num_c1: 6.895143982427237E-015 + so4_c2: 142214.467855250 + soa_c2: 3.181118913367564E-014 + ncl_c2: 2.996638282752921E-015 + mom_c2: 8.055333842809433E-016 + num_c2: 5.202632275847699E-017 + dst_c3: 464.524744048298 + ncl_c3: 8.814810290885256E-012 + so4_c3: 1.417415096039318E-012 + bc_c3: 1.284022582681906E-013 + pom_c3: 1.081226492056768E-015 + soa_c3: 3.292708392236485E-015 + mom_c3: 2.584587804061707E-014 + num_c3: 2.324887367507579E-016 + pom_c4: 6.934165370527951E-014 + bc_c4: 2.703924752644086E-026 + mom_c4: 3.684880329580097E-027 + num_c4: 1.049556651713384E-032 + + frzimm: 5.651860156917002E-006 4.477615226393836E-008 5.607084004653063E-006 + frzcnt: 0.000000000000000E+000 + frzdep: 0.000000000000000E+000 + + BC: 2.686569135836302E-003 6.811694223739704E-014* 4.327927689273498E-012* + 8.647238075040541E-013* 6.734058317716400E-012* 2.685810991448475E-014* + 8.239303776951524E-014* 2.218253512777307E-015**** 477478.884677930* + 1.000000000000000E-006* + + --BC: 2.686569135836302E-003 6.734058317716400E-012 477478.884677930 4.327927689273498E-012 8.647238075040541E-013 8.239303776951524E-014 6.811694223739704E-014 + 2.685810991448475E-014 1.000000000000000E-006 + + BC:0.00209583 : 6.81169e-14 : 4.32793e-12 : 8.64724e-13 : 6.73406e-12 : 2.68581e-14 : 8.23930e-14 : 2.21825e-15 : 372488. : 1.00000e-06 + + */ \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 3c01f469092..88b442d7c2f 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -653,11 +653,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -678,21 +678,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);// PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 2b33c95a517..95da48b7fa4 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,8 +12,8 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +set (ATM_TIME_STEP 600) set (RUN_T0 2021-10-12-45000) diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index 4941b2d259b..bedc9c583c2 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -25,23 +25,16 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file #we should get the following variables from other processes pbl_height : 1.0 - p_mid: 86389.8729712384 - omega: -0.103206160508178 - T_mid: 279.796493114264 - dgnum: 1.326982247461267E-007 - eddy_diff_heat: 2.82379273906116 - cldfrac_liq: 0.999966505294370 - cldfrac_liq_prev: 0.999999418697814 - w_variance: 1.548741246470719E-002 + dgnum: 1e-3 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] -... +... \ No newline at end of file From 9a08cc66f4ea654e42958701c89d756fc465b538 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Apr 2024 12:15:24 -0700 Subject: [PATCH 078/142] Fakes interfaces values for kvh and wsec; fixes test and namelist wsubmin --- .../cime_config/namelist_defaults_scream.xml | 3 +- .../mam/eamxx_mam_aci_process_interface.cpp | 262 ++++++++++++------ .../mam/eamxx_mam_aci_process_interface.hpp | 10 +- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 2 + 4 files changed, 181 insertions(+), 96 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 8aa4c0ea6f7..2eda717ee06 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -236,8 +236,9 @@ be lost if SCREAM_HACK_XML is not enabled. - 0.001 + 0.001 + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 1d47fdcc559..53e72fc8dcb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -185,11 +185,11 @@ void compute_nucleate_ice_tendencies( // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { // const int icol = team.league_rank(); for(int icol = 0; icol < 5; ++icol) { - //std::cout<<""<("w_variance", scalar3d_layout_int, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -779,7 +779,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -861,8 +861,7 @@ void MAMAci::set_grids( n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] - add_field("num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, - grid_name); + add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) @@ -975,11 +974,11 @@ void MAMAci::initialize_impl(const RunType run_type) { // ------------------------------------------------------------------------ // Input fields read in from IC file, namelist or other processes // ------------------------------------------------------------------------ - w_sec_ = get_field_in("w_variance").get_view(); + w_sec_mid_ = get_field_in("w_variance").get_view(); dgnum_ = get_field_in("dgnum").get_view(); liqcldf_ = get_field_in("cldfrac_liq").get_view(); liqcldf_prev_ = get_field_in("cldfrac_liq_prev").get_view(); - kvh_ = get_field_in("eddy_diff_heat").get_view(); + kvh_mid_ = get_field_in("eddy_diff_heat").get_view(); // store fields only to be converted to dry mmrs in wet_atm_ wet_atm_.qv = get_field_in("qv").get_view(); @@ -1027,8 +1026,8 @@ void MAMAci::initialize_impl(const RunType run_type) { naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); - naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); - qcld_ = get_field_out("qcld").get_view(); + naai_ = get_field_out("ni_activated").get_view(); + qcld_ = get_field_out("qcld").get_view(); ptend_q_output_ = get_field_out("ptend_q").get_view(); tendnd_ = get_field_out("tendnd").get_view(); factnum_ = get_field_out("factnum").get_view(); @@ -1116,10 +1115,17 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); // nact : fractional aero. number activation rate [/s] - // mact : fractional aero. mass activation rate [/s] Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // mact : fractional aero. mass activation rate [/s] Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + // Eddy diffusivity of heat at the interfaces + Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); + + // Vertical velocity variance at the interfaces + Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); + mam4::AeroConfig aero_config; // configure the nucleation parameterization mam4::NucleateIce::Config nucleate_ice_config; @@ -1138,11 +1144,13 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - //for (int kk=0; kk<72; ++kk){ - //std::cout << "num_cb_accum_top:"<::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, + // nlev_); KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // preprocess input -- needs a scan for the calculation of local derivied @@ -1150,60 +1158,94 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); const int kb = 62; compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); - //std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; - //std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; - //std::cout << "wsec:" << w_sec_(0, kb) << std::endl; - //std::cout << "w0:" << w0_(0, kb) << std::endl; - //std::cout << " rho: " << rho_(0, kb) << std::endl; + // std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; + // std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; + // std::cout << "wsec:" << w_sec_(0, kb) << std::endl; + // std::cout << "w0:" << w0_(0, kb) << std::endl; + // std::cout << " rho: " << rho_(0, kb) << std::endl; + + // compute vertical velocity variance at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + } + w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + } - compute_tke_using_w_sec(team_policy, w_sec_, nlev_, + compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output tke_); - //std::cout << "TKE:" << tke_(0, kb) << std::endl; + // std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. const Real &wsubmin = ACIRuntime_.wsubmin; // runtime variable compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); - //std::cout << "WSUB:" << wsub_(0, kb) << std::endl; - //std::cout << "WICE:" << wsubice_(0, kb) << std::endl; - //std::cout << "WSIG:" << wsig_(0, kb) << std::endl; + // std::cout << "WSUB:" << wsub_(0, kb) << std::endl; + // std::cout << "WICE:" << wsubice_(0, kb) << std::endl; + // std::cout << "WSIG:" << wsig_(0, kb) << std::endl; Kokkos::fence(); // wait for wsig_ to be computed. - /* wo 3.343177351577457E-004 - rho: 1.28186433384113 - TKE: 1.666465903934813E-002 6.097664104873541E-002 - wsub: 0.160873967324407 - wsubi: 0.200000000000000 - wsig: 0.160873967324407 - dgnum_ait: 4.362354500358337E-008 - naai: 11464.5957222634 - naai_hom: 29838.6879763933 - */ - compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - const Real dgnum_ait_e3sm[72] = { 0.20877713336487552E-007, 0.21782230353342090E-007, 0.21688324003865861E-007, 0.21112855042342451E-007, 0.19162058462939536E-007, 0.18102979880838476E-007, 0.17906980715477606E-007, 0.20271254074583327E-007, 0.22698983422181942E-007, 0.24134835117044986E-007, 0.25498156808001372E-007, 0.29796738799905547E-007, 0.35822987394021908E-007, 0.41170963764365215E-007, 0.44892726528330642E-007, 0.47217231342203107E-007, 0.48928661807108766E-007, 0.50170939816128735E-007, 0.51078750853732200E-007, 0.52247333465736065E-007, 0.53190758580174931E-007, 0.53576491941850044E-007, 0.53915614473890715E-007, 0.54510964775236826E-007, 0.55643231691556703E-007, 0.57057811112589899E-007, 0.58177383586181116E-007, 0.58209849180850108E-007, 0.57976751598840998E-007, 0.52000000000000002E-007, 0.50728746567226150E-007, 0.49119902704480870E-007, 0.48212162162050883E-007, 0.49227715213506454E-007, 0.46876827233752246E-007, 0.45360603896257791E-007, 0.49986783979004747E-007, 0.51186879246229022E-007, 0.50009353247048599E-007, 0.48250264542204811E-007, 0.47560278748093609E-007, 0.48298089720730957E-007, 0.49095935613468768E-007, 0.49493024126912931E-007, 0.50250797590476007E-007, 0.51949267668322422E-007, 0.53778727208416418E-007, 0.53563593301099588E-007, 0.51218136771199298E-007, 0.43171429694325200E-007, 0.39019610039033895E-007, 0.36175109143257051E-007, 0.42731638777892750E-007, 0.38060728507221777E-007, 0.44046323901481340E-007, 0.39216732751330010E-007, 0.34842233953609988E-007, 0.34068804733226066E-007, 0.30636043694263528E-007, 0.28302341686131413E-007, 0.33023014309036320E-007, 0.34745748365385196E-007, 0.43623545003583371E-007, 0.48206451795644064E-007, 0.49854490325455530E-007, 0.50346335647724146E-007, 0.50661560988561763E-007, 0.50986261962838767E-007, 0.51256955985111086E-007, 0.51482578449096488E-007, 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < 218; ++icol){ - for(int kk = 0; kk < 72; ++kk){ - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - }} + const Real dgnum_ait_e3sm[72] = { + 0.20877713336487552E-007, 0.21782230353342090E-007, + 0.21688324003865861E-007, 0.21112855042342451E-007, + 0.19162058462939536E-007, 0.18102979880838476E-007, + 0.17906980715477606E-007, 0.20271254074583327E-007, + 0.22698983422181942E-007, 0.24134835117044986E-007, + 0.25498156808001372E-007, 0.29796738799905547E-007, + 0.35822987394021908E-007, 0.41170963764365215E-007, + 0.44892726528330642E-007, 0.47217231342203107E-007, + 0.48928661807108766E-007, 0.50170939816128735E-007, + 0.51078750853732200E-007, 0.52247333465736065E-007, + 0.53190758580174931E-007, 0.53576491941850044E-007, + 0.53915614473890715E-007, 0.54510964775236826E-007, + 0.55643231691556703E-007, 0.57057811112589899E-007, + 0.58177383586181116E-007, 0.58209849180850108E-007, + 0.57976751598840998E-007, 0.52000000000000002E-007, + 0.50728746567226150E-007, 0.49119902704480870E-007, + 0.48212162162050883E-007, 0.49227715213506454E-007, + 0.46876827233752246E-007, 0.45360603896257791E-007, + 0.49986783979004747E-007, 0.51186879246229022E-007, + 0.50009353247048599E-007, 0.48250264542204811E-007, + 0.47560278748093609E-007, 0.48298089720730957E-007, + 0.49095935613468768E-007, 0.49493024126912931E-007, + 0.50250797590476007E-007, 0.51949267668322422E-007, + 0.53778727208416418E-007, 0.53563593301099588E-007, + 0.51218136771199298E-007, 0.43171429694325200E-007, + 0.39019610039033895E-007, 0.36175109143257051E-007, + 0.42731638777892750E-007, 0.38060728507221777E-007, + 0.44046323901481340E-007, 0.39216732751330010E-007, + 0.34842233953609988E-007, 0.34068804733226066E-007, + 0.30636043694263528E-007, 0.28302341686131413E-007, + 0.33023014309036320E-007, 0.34745748365385196E-007, + 0.43623545003583371E-007, 0.48206451795644064E-007, + 0.49854490325455530E-007, 0.50346335647724146E-007, + 0.50661560988561763E-007, 0.50986261962838767E-007, + 0.51256955985111086E-007, 0.51482578449096488E-007, + 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < 218; ++icol) { + for(int kk = 0; kk < 72; ++kk) { + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + } + } Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - //std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; - // Compute Ice nucleation - // NOTE: The Fortran version uses "ast" for cloud fraction which is equivalent - // to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct + // std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; + // Compute Ice nucleation + // NOTE: The Fortran version uses "ast" for cloud fraction which is + // equivalent to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct compute_nucleate_ice_tendencies( nucleate_ice_, team_policy, dry_atm_, dry_aero_, wsubice_, aitken_dry_dia_, nlev_, dt, @@ -1215,8 +1257,8 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); - //std::cout << "naai_hom_:" << naai_hom_(0, kb) << std::endl; - //std::cout << "naai_:" << naai_(0, kb) << std::endl; + // std::cout << "naai_hom_:" << naai_hom_(0, kb) << std::endl; + // std::cout << "naai_:" << naai_(0, kb) << std::endl; // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- @@ -1238,11 +1280,19 @@ void MAMAci::run_impl(const double dt) { compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); - //std::cout << "rpdel_:" << rpdel_(0, kb) << std::endl; + // std::cout << "rpdel_:" << rpdel_(0, kb) << std::endl; Kokkos::fence(); // wait for rpdel_ to be computed. - call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, + // compute eddy diffusivity of heat at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + kvh_int_(icol, kk) = kvh_mid_(icol, kk); + } + kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + } + + call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output qqcw_fld_work_, ptend_q_, factnum_, tendnd_, @@ -1252,17 +1302,18 @@ void MAMAci::run_impl(const double dt) { raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - - //std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, kb)<<" : "<( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1278,10 +1329,13 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); - std::cout << "hetfrz_immersion_nucleation_tend_:" << hetfrz_immersion_nucleation_tend_(0, kb)<< std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" << hetfrz_contact_nucleation_tend_(0, kb)<< std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" << hetfrz_depostion_nucleation_tend_(0, kb)<< std::endl; - + std::cout << "hetfrz_immersion_nucleation_tend_:" + << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" + << hetfrz_contact_nucleation_tend_(0, kb) << std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" + << hetfrz_depostion_nucleation_tend_(0, kb) << std::endl; + Kokkos::fence(); // wait before returning to calling function } @@ -1290,9 +1344,45 @@ void MAMAci::finalize_impl() { } } // namespace scream - //tendnd: -839.404918218856 7289315.36108503 7792958.31201634 1.666666666666667E-003 - //factnum: 0.998556176544263 0.861689826773470 0.999999999974140 0.000000000000000E+000 - /*nctend_mixnuc: -839.404918218856 + +/* +FORTRAN OUTPUT: + + ----- OUPUT AT time step: 6 6 + wo 3.343177351577457E-004 + rho: 1.28186433384113 + TKE: 1.666465903934813E-002 6.097664104873541E-002 + wsub: 0.160873967324407 + wsubi: 0.200000000000000 + wsig: 0.160873967324407 + dgnum_ait: 4.362354500358337E-008 + naai: 11464.5957222634 + naai_hom: 29838.6879763933 + aer_cb: so4_c1 4.327927689273498E-012 so4_c1 4.327927689273498E-012 + aer_cb: bc_c1 6.811694223739704E-014 bc_c1 6.811694223739704E-014 + aer_cb: pom_c1 8.647238075040541E-013 pom_c1 8.647238075040541E-013 + aer_cb: soa_c1 6.734058317716400E-012 soa_c1 6.734058317716400E-012 + aer_cb: dst_c1 8.239303776951524E-014 dst_c1 8.239303776951524E-014 + aer_cb: ncl_c1 2.685810991448475E-014 ncl_c1 2.685810991448475E-014 + aer_cb: mom_c1 2.218253512777307E-015 mom_c1 2.218253512777307E-015 + aer_cb: num_c1 477478.884677930 num_c1 477478.884677930 + aer_cb: dst_c3 2.869197497896834E-012 dst_c3 2.869197497896834E-012 + aer_cb: ncl_c3 4.636857908110365E-013 ncl_c3 4.636857908110365E-013 + aer_cb: so4_c3 4.375223642985256E-014 so4_c3 4.375223642985256E-014 + aer_cb: bc_c3 3.674860666928688E-016 bc_c3 3.674860666928688E-016 + aer_cb: pom_c3 1.114802999982066E-015 pom_c3 1.114802999982066E-015 + aer_cb: soa_c3 8.915776993183282E-015 soa_c3 8.915776993183282E-015 + aer_cb: mom_c3 7.802642470782658E-017 mom_c3 7.802642470782658E-017 + aer_cb: num_c3 151.593971049442 num_c3 151.593971049442 + aer_cb: bc_c4 1.127495264613331E-027 bc_c4 1.127495264613331E-027 + aer_cb: pom_c4 8.204883715624573E-027 pom_c4 8.204883715624573E-027 + aer_cb: mom_c4 3.249069782816687E-033 mom_c4 3.249069782816687E-033 + aer_cb: num_c4 1.345885758586041E-013 num_c4 1.345885758586041E-013 + tendnd: -839.404918218856 7289315.36108503 7792958.31201634 + 1.666666666666667E-003 + factnum: 0.998556176544263 0.861689826773470 + 0.999999999974140 0.000000000000000E+000 + nctend_mixnuc: -839.404918218856 O3_tend: 0.000000000000000E+000 H2O2_tend: 0.000000000000000E+000 H2SO4_tend: 0.000000000000000E+000 @@ -1306,12 +1396,12 @@ void MAMAci::finalize_impl() { dst_a1_tend: -2.194060414083922E-016 ncl_a1_tend: -6.981124745267083E-017 mom_a1_tend: -5.662690535048673E-018 - num_a1_tend: -1626.38730532537 + num_a1_tend: -1626.38730532537 so4_a2_tend: -2.548707408341951E-017 soa_a2_tend: -1.929990276139139E-018 ncl_a2_tend: -6.855087738119723E-019 mom_a2_tend: -4.571691604185304E-020 - num_a2_tend: -111.890330245159 + num_a2_tend: -111.890330245159 dst_a3_tend: -7.609611333715698E-015 ncl_a3_tend: -1.256350252705383E-015 so4_a3_tend: -1.320988655880598E-016 @@ -1319,12 +1409,12 @@ void MAMAci::finalize_impl() { pom_a3_tend: -3.309939677824050E-018 soa_a3_tend: -2.767668335981830E-017 mom_a3_tend: -2.254911604324925E-019 - num_a3_tend: -0.406788905791186 + num_a3_tend: -0.406788905791186 pom_a4_tend: -2.375070578317096E-017 bc_a4_tend: 4.082479713528180E-017 mom_a4_tend: -4.901356204764327E-023 - num_a4_tend: -34.4232851017138 - so4_c1: 1322558.40616450 + num_a4_tend: -34.4232851017138 + so4_c1: 1322558.40616450 pom_c1: 1.233997159684093E-011 soa_c1: 2.393397505793018E-012 bc_c1: 1.872520079842212E-011 @@ -1332,12 +1422,12 @@ void MAMAci::finalize_impl() { ncl_c1: 2.530385880698310E-013 mom_c1: 8.315014074126540E-014 num_c1: 6.895143982427237E-015 - so4_c2: 142214.467855250 + so4_c2: 142214.467855250 soa_c2: 3.181118913367564E-014 ncl_c2: 2.996638282752921E-015 mom_c2: 8.055333842809433E-016 num_c2: 5.202632275847699E-017 - dst_c3: 464.524744048298 + dst_c3: 464.524744048298 ncl_c3: 8.814810290885256E-012 so4_c3: 1.417415096039318E-012 bc_c3: 1.284022582681906E-013 @@ -1349,19 +1439,9 @@ void MAMAci::finalize_impl() { bc_c4: 2.703924752644086E-026 mom_c4: 3.684880329580097E-027 num_c4: 1.049556651713384E-032 - - frzimm: 5.651860156917002E-006 4.477615226393836E-008 5.607084004653063E-006 + + frzimm: 5.651860156917002E-006 frzcnt: 0.000000000000000E+000 frzdep: 0.000000000000000E+000 - BC: 2.686569135836302E-003 6.811694223739704E-014* 4.327927689273498E-012* - 8.647238075040541E-013* 6.734058317716400E-012* 2.685810991448475E-014* - 8.239303776951524E-014* 2.218253512777307E-015**** 477478.884677930* - 1.000000000000000E-006* - - --BC: 2.686569135836302E-003 6.734058317716400E-012 477478.884677930 4.327927689273498E-012 8.647238075040541E-013 8.239303776951524E-014 6.811694223739704E-014 - 2.685810991448475E-014 1.000000000000000E-006 - - BC:0.00209583 : 6.81169e-14 : 4.32793e-12 : 8.64724e-13 : 6.73406e-12 : 2.68581e-14 : 8.23930e-14 : 2.21825e-15 : 372488. : 1.00000e-06 - */ \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 3159fa3f0be..865cdcdc0bf 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -67,9 +67,10 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d nimey_; view_2d naai_hom_; view_2d naai_; + view_2d kvh_int_; // Eddy diffusivity of heat at the interfaces const_view_2d liqcldf_; const_view_2d liqcldf_prev_; - const_view_2d kvh_; + const_view_2d kvh_mid_; // Eddy diffusivity of heat at the midpoints view_2d cloud_frac_; view_2d cloud_frac_prev_; @@ -120,9 +121,10 @@ class MAMAci final : public scream::AtmosphereProcess { const int top_lev_ = 6; // local atmospheric state column variables - const_view_2d pdel_; // pressure thickess of layer [Pa] - view_2d rpdel_; // Inverse of pdel_ - const_view_2d w_sec_; // Vertical velocity variance + const_view_2d pdel_; // pressure thickess of layer [Pa] + view_2d rpdel_; // Inverse of pdel_ + const_view_2d w_sec_mid_; // Vertical velocity variance at midpoints + view_2d w_sec_int_; // Vertical velocity variance at interfaces // number of horizontal columns and vertical levels int ncol_, nlev_; diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index f41b961a4f5..4a1c80ba7ca 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -32,6 +32,8 @@ atmosphere_processes: c_diag_3rd_mom: 7.0 Ckh: 0.1 Ckm: 0.1 + mam4_aci: + wsubmin: 0.001 grids_manager: Type: Mesh Free From fa3cc9189ed0784b5a94a2e1c11d99e1dc9ab3a0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Apr 2024 12:21:26 -0700 Subject: [PATCH 079/142] Moves all resize views codes in set_grid --- .../mam/eamxx_mam_aci_process_interface.cpp | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 53e72fc8dcb..c4555877df8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -693,6 +693,37 @@ void MAMAci::set_grids( for(int i = 0; i < dropmix_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + // These are temp arrays formatted like mam4xx wants. + // Not sure if there is a way to do this with scream. + Kokkos::resize(coltend_[i], ncol_, nlev_); + Kokkos::resize(coltend_cw_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { + Kokkos::resize(ptend_q_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); + Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); + } + } + + for(int i = 0; i < 42; ++i) + Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); + + // nact : fractional aero. number activation rate [/s] + Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // mact : fractional aero. mass activation rate [/s] + Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // Eddy diffusivity of heat at the interfaces + Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); + + // Vertical velocity variance at the interfaces + Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); + // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -1095,37 +1126,6 @@ void MAMAci::initialize_impl(const RunType run_type) { state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, mam4::aero_model::pcnst); - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - // These are temp arrays formatted like mam4xx wants. - // Not sure if there is a way to do this with scream. - Kokkos::resize(coltend_[i], ncol_, nlev_); - Kokkos::resize(coltend_cw_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - Kokkos::resize(ptend_q_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { - Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); - Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); - } - } - - for(int i = 0; i < 42; ++i) - Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); - - // nact : fractional aero. number activation rate [/s] - Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); - - // mact : fractional aero. mass activation rate [/s] - Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); - - // Eddy diffusivity of heat at the interfaces - Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); - - // Vertical velocity variance at the interfaces - Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); - mam4::AeroConfig aero_config; // configure the nucleation parameterization mam4::NucleateIce::Config nucleate_ice_config; From 4d6ec50ba9399d79973e61ecd649a67603c5a0dd Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Apr 2024 17:28:47 -0700 Subject: [PATCH 080/142] Connects ni_activated with P3 --- .../mam/eamxx_mam_aci_process_interface.cpp | 88 ++++++++++++++++++- .../shoc_cldfrac_mam4_aci_p3/CMakeLists.txt | 3 +- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 2 +- 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index c4555877df8..6a175660f86 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1172,11 +1172,51 @@ void MAMAci::run_impl(const double dt) { // std::cout << " rho: " << rho_(0, kb) << std::endl; // compute vertical velocity variance at the interfaces + const Real w_sec_e3sm[73] = { + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.44201892319518146E-003, + 0.77315620137962326E-003, 0.24806301482800117E-002, + 0.11109772692898754E-001, 0.40651094032490273E-001, + 0.82156694426095800E-001, 0.12207124453993526E+000, + 0.15516728994634110E+000, 0.17775318086169636E+000, + 0.18549817250146838E+000, 0.17184548286554119E+000, + 0.12741230682196053E+000, 0.65495229516041628E-001, + 0.26909155217660592E-001}; for(int icol = 0; icol < ncol_; ++icol) { for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + w_sec_int_(icol, kk) = w_sec_e3sm[kk]; + // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); } - w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; } compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, @@ -1251,6 +1291,7 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_, nlev_, dt, // output nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); + std::cout << "naai:" << naai_(0, 62) << std::endl; store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, @@ -1284,12 +1325,51 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. + const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, + 0.75991761081225006E-002, 0.12291092068185365E-001, + 0.11484807652762415E-001, 0.10856880396302943E-001, + 0.10500384508819637E-001, 0.10361486171738229E-001, + 0.10333325067964508E-001, 0.10258838031435397E-001, + 0.10027325248446619E-001, 0.97784259072973521E-002, + 0.96611845055866539E-002, 0.96249746122327937E-002, + 0.95773431515696512E-002, 0.95180614513688099E-002, + 0.94713233348487150E-002, 0.94503864489758338E-002, + 0.94536294366578833E-002, 0.94575972194308883E-002, + 0.94403767489615684E-002, 0.93975694769176284E-002, + 0.93322843554751022E-002, 0.92777070192527501E-002, + 0.92456776697171228E-002, 0.92266924824142716E-002, + 0.92123025773060436E-002, 0.91888715633294191E-002, + 0.91516797753615851E-002, 0.90958299606649744E-002, + 0.89988037524983237E-002, 0.88220984587642423E-002, + 0.85231270833157156E-002, 0.81397522619395188E-002, + 0.79160421807845088E-002, 0.81206851117902653E-002, + 0.86526891616674779E-002, 0.91682975412125615E-002, + 0.96043394254592580E-002, 0.10033689085881327E-001, + 0.10428656694074272E-001, 0.10715913043864789E-001, + 0.10919631245454951E-001, 0.11250937075285789E-001, + 0.11829292157343831E-001, 0.12413311776454055E-001, + 0.12851317662157077E-001, 0.13175523677700330E-001, + 0.13224182907540188E-001, 0.13085937680733115E-001, + 0.12615055546741534E-001, 0.11995423733019836E-001, + 0.12346556881757400E-001, 0.13433752971524651E-001, + 0.13904308240950175E-001, 0.13539811748121957E-001, + 0.12555099320041433E-001, 0.11519643673351362E-001, + 0.11414071302852231E-001, 0.13409756835238139E-001, + 0.24071962815959351E-001, 0.75489419450816414E-001, + 0.62082011878960308E+000, 0.63952862312816796E+001, + 0.16226857944175123E+002, 0.21882852534279891E+002, + 0.24966173574402408E+002, 0.25710753126453692E+002, + 0.24069881024271943E+002, 0.19743922403487922E+002, + 0.98667814246712027E+001, 0.25633359450143991E+001, + 0.14682471685037493E+001}; // compute eddy diffusivity of heat at the interfaces for(int icol = 0; icol < ncol_; ++icol) { for(int kk = 0; kk < nlev_; ++kk) { - kvh_int_(icol, kk) = kvh_mid_(icol, kk); + // kvh_int_(icol, kk) = kvh_mid_(icol, kk); + kvh_int_(icol, kk) = kvh_e3sm[kk]; } - kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; } call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt index 946d68cb750..81eaa3ae3e4 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt @@ -13,7 +13,8 @@ CreateADUnitTest(${TEST_BASE_NAME} # Set AD configurable options set (ATM_TIME_STEP 1800) -SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +#SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +SetVarDependingOnTestSize(NUM_STEPS 1 1 1) # 1h 4h 24h set (RUN_T0 2021-10-12-45000) # Determine num subcycles needed to keep shoc dt<=300s diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index 4a1c80ba7ca..5cc2fb8fa65 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -47,7 +47,7 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} #variable required for shoc From 4ea07a3b6ed15c1aa1fa70fc5cb1d4da35dcc3ce Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 09:59:25 -0700 Subject: [PATCH 081/142] nctend or tendnd is not connected with P3 --- .../mam/eamxx_mam_aci_process_interface.cpp | 134 ++++++++++-------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 6a175660f86..a4d7fc5c920 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -181,62 +181,63 @@ void compute_nucleate_ice_tendencies( // from ice nucleation //------------------------------------------------------------- - // Kokkos::parallel_for( - // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - // const int icol = team.league_rank(); - for(int icol = 0; icol < 5; ++icol) { - // std::cout<<""<(mam4::ModeIndex::Aitken); - diags.dry_geometric_mean_diameter_i[aitken_idx] = - ekat::subview(aitken_dry_dia, icol); - - // These are the fields that are updated. Taking subviews means that - // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated - // in nucleate_ice.compute_tendencies. - diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); - diags.icenuc_num_immfrz = ekat::subview(niim, icol); - diags.icenuc_num_depnuc = ekat::subview(nidep, icol); - diags.icenuc_num_meydep = ekat::subview(nimey, icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); // not used - const mam4::AeroConfig aero_config; - const Real t = 0; // not used - nucleate_ice.compute_tendencies(aero_config, /*team,*/ t, dt, haero_atm, - surf, progs, diags, tends); - //}); - } + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // for(int icol = 0; icol < 5; ++icol) { + // std::cout<<""<(mam4::ModeIndex::Aitken); + diags.dry_geometric_mean_diameter_i[aitken_idx] = + ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated + // in nucleate_ice.compute_tendencies. + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); // not used + const mam4::AeroConfig aero_config; + const Real t = 0; // not used + nucleate_ice.compute_tendencies(aero_config, /*team,*/ t, dt, haero_atm, + surf, progs, diags, tends); + }); + //} } KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( @@ -906,7 +907,7 @@ void MAMAci::set_grids( add_field("ptend_q", scalar4d_layout_nconst_mid, q_unit, grid_name); // tendency in droplet number mixing ratio [#/kg/s] - add_field("tendnd", scalar3d_layout_mid, n_unit / s, grid_name); + add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); // activation fraction for aerosol number [fraction] add_field("factnum", scalar4d_layout_mid, nondim, grid_name); @@ -1060,7 +1061,7 @@ void MAMAci::initialize_impl(const RunType run_type) { naai_ = get_field_out("ni_activated").get_view(); qcld_ = get_field_out("qcld").get_view(); ptend_q_output_ = get_field_out("ptend_q").get_view(); - tendnd_ = get_field_out("tendnd").get_view(); + tendnd_ = get_field_out("nc_nuceat_tend").get_view(); factnum_ = get_field_out("factnum").get_view(); ndropcol_ = get_field_out("ndropcol").get_view(); ndropmix_ = get_field_out("ndropmix").get_view(); @@ -1275,8 +1276,8 @@ void MAMAci::run_impl(const double dt) { 0.50661560988561763E-007, 0.50986261962838767E-007, 0.51256955985111086E-007, 0.51482578449096488E-007, 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < 218; ++icol) { - for(int kk = 0; kk < 72; ++kk) { + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; } } @@ -1291,7 +1292,12 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_, nlev_, dt, // output nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); - std::cout << "naai:" << naai_(0, 62) << std::endl; + + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + naai_(icol, kk) =9876.0; + } + } store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, @@ -1382,7 +1388,11 @@ void MAMAci::run_impl(const double dt) { raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + tendnd_(icol, kk) =1.2345; + } + } // std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, // kb)<<" : "< Date: Sat, 20 Apr 2024 10:09:54 -0700 Subject: [PATCH 082/142] Adds a branched mam4xx submodule with ndrop validation changes --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 7b0be56f4c3..0f80195e0ff 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 7b0be56f4c3f164259c6a9d8a0d16d6f5b72f3ae +Subproject commit 0f80195e0ff30d00d9a9b651e63a4ab4e314a242 From e78203fe528debade7cfc1b54135717089bcf87d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 19:01:30 -0700 Subject: [PATCH 083/142] Removes ACIRuntime namelist struct and used wsubmin on its own --- .../mam/eamxx_mam_aci_process_interface.cpp | 139 +++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 9 -- 3 files changed, 74 insertions(+), 76 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index a4d7fc5c920..a6eb50fe82e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -671,60 +671,15 @@ MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) void MAMAci::set_grids( const std::shared_ptr grids_manager) { // set grid for all the inputs and outputs - grid_ = grids_manager->get_grid("Physics"); // Use physics grid - const auto &grid_name = grid_->name(); // Name of the grid + // use physics grid + grid_ = grids_manager->get_grid("Physics"); + + // Name of the grid + const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - // Allocate memory for the class members - // Kokkos::resize only works on host to allocates memory - Kokkos::resize(rho_, ncol_, nlev_); - Kokkos::resize(w0_, ncol_, nlev_); - Kokkos::resize(tke_, ncol_, nlev_ + 1); - Kokkos::resize(wsub_, ncol_, nlev_); - Kokkos::resize(wsubice_, ncol_, nlev_); - Kokkos::resize(wsig_, ncol_, nlev_); - Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(cloud_frac_, ncol_, nlev_); - Kokkos::resize(cloud_frac_prev_, ncol_, nlev_); - Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); - Kokkos::resize(rpdel_, ncol_, nlev_); - - for(int i = 0; i < dropmix_scratch_; ++i) { - Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - // These are temp arrays formatted like mam4xx wants. - // Not sure if there is a way to do this with scream. - Kokkos::resize(coltend_[i], ncol_, nlev_); - Kokkos::resize(coltend_cw_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - Kokkos::resize(ptend_q_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { - Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); - Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); - } - } - - for(int i = 0; i < 42; ++i) - Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); - - // nact : fractional aero. number activation rate [/s] - Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); - - // mact : fractional aero. mass activation rate [/s] - Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); - - // Eddy diffusivity of heat at the interfaces - Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); - - // Vertical velocity variance at the interfaces - Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); - // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -780,7 +735,7 @@ void MAMAci::set_grids( // planetary boundary layer height add_field("pbl_height", scalar2d_layout_col, m, grid_name); - // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process + // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); auto m2 = m * m; @@ -789,7 +744,7 @@ void MAMAci::set_grids( s2.set_string("s^2"); // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time - // step Since, we are using SE dycore, itim_old is 1 which is equivalent to + // step. Since, we are using SE dycore, itim_old is 1 which is equivalent to // the current time step. For other dycores (such as EUL), it may be different // and we might need to revisit this. @@ -797,7 +752,7 @@ void MAMAci::set_grids( // SHOC provides it at the midpoints. Verify how it is being used. // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -811,7 +766,8 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + // FIXME: It is at mid level in EAMxx, we need to compute it at the interfaces + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -907,13 +863,14 @@ void MAMAci::set_grids( add_field("ptend_q", scalar4d_layout_nconst_mid, q_unit, grid_name); // tendency in droplet number mixing ratio [#/kg/s] - add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); + add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, + grid_name); // activation fraction for aerosol number [fraction] add_field("factnum", scalar4d_layout_mid, nondim, grid_name); // NOTE: Here is a series of internal dropmixnuc variables; - // maybe we should move them to diagnostics later + // maybe we should move them to diagnostics later (FIXME) // cloud droplet number mixing ratio [#/kg] add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); @@ -1001,7 +958,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // ## Runtime options // ------------------------------------------------------------------------ - ACIRuntime_.wsubmin = m_params.get("wsubmin"); + wsubmin_ = m_params.get("wsubmin"); // ------------------------------------------------------------------------ // Input fields read in from IC file, namelist or other processes @@ -1120,6 +1077,54 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // Allocate memory for the class members + // (Kokkos::resize only works on host to allocates memory) + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + Kokkos::resize(tke_, ncol_, nlev_ + 1); + Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(wsubice_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(cloud_frac_, ncol_, nlev_); + Kokkos::resize(cloud_frac_prev_, ncol_, nlev_); + Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); + Kokkos::resize(rpdel_, ncol_, nlev_); + + for(int i = 0; i < dropmix_scratch_; ++i) { + Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + // These are temp arrays formatted like mam4xx wants. + // Not sure if there is a way to do this with scream. + // FIXME: Do we need these? + Kokkos::resize(coltend_[i], ncol_, nlev_); + Kokkos::resize(coltend_cw_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { + Kokkos::resize(ptend_q_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); + Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); + } + } + + for(int i = 0; i < 42; ++i) + Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); + + // nact : fractional aero. number activation rate [/s] + Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // mact : fractional aero. mass activation rate [/s] + Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // Eddy diffusivity of heat at the interfaces + Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); + + // Vertical velocity variance at the interfaces + Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); // Allocate work arrays for(int icnst = 0; icnst < mam4::ndrop::ncnst_tot; ++icnst) { qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); @@ -1226,8 +1231,7 @@ void MAMAci::run_impl(const double dt) { // std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. - const Real &wsubmin = ACIRuntime_.wsubmin; // runtime variable - compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, + compute_subgrid_scale_velocities(team_policy, tke_, wsubmin_, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); // std::cout << "WSUB:" << wsub_(0, kb) << std::endl; @@ -1295,7 +1299,7 @@ void MAMAci::run_impl(const double dt) { for(int icol = 0; icol < ncol_; ++icol) { for(int kk = 0; kk < nlev_; ++kk) { - naai_(icol, kk) =9876.0; + naai_(icol, kk) = 9876.0; } } @@ -1388,11 +1392,7 @@ void MAMAci::run_impl(const double dt) { raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - tendnd_(icol, kk) =1.2345; - } - } + // std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, // kb)<<" : "< 5.65186e-06) { + std::cout << "SOmethign changed!!!! :" + << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; + exit(1); + } std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; + << hetfrz_immersion_nucleation_tend_(0, kb) << ":" + << hetfrz_immersion_nucleation_tend_(0, 0) << std::endl; std::cout << "hetfrz_contact_nucleation_tend_:" << hetfrz_contact_nucleation_tend_(0, kb) << std::endl; std::cout << "hetfrz_depostion_nucleation_tend_:" diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 865cdcdc0bf..297078c8727 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -36,7 +36,7 @@ class MAMAci final : public scream::AtmosphereProcess { // micriphysics and optics codes // ACI runtime ( or namelist) options - mam_coupling::MAM4ACIRuntime ACIRuntime_; + Real wsubmin_; // rho is air density [kg/m3] view_2d rho_; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 88b442d7c2f..25cc9d5de9d 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -252,15 +252,6 @@ const char* gas_mmr_field_name(const int gas) { return const_cast(gas_species_name(gas)); } - // This struct stores runtime options for MAM4xx-ACI - struct MAM4ACIRuntime { - MAM4ACIRuntime() = default; - // Runtime options for MAM4xx aerosol cloud interactions code - - Real wsubmin; //Minimum diagnostic sub-grid vertical velocity - }; - - // This type stores multi-column views related specifically to the wet // atmospheric state used by EAMxx. struct WetAtmosphere { From e34a548a227f6544bd99f3b95b3858bbeba42639 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 20:36:58 -0700 Subject: [PATCH 084/142] Clean up and moves validation intput output in functions --- .../mam/eamxx_mam_aci_process_interface.cpp | 410 +++++++++--------- 1 file changed, 211 insertions(+), 199 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index a6eb50fe82e..67184b7466d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -3,6 +3,194 @@ namespace scream { namespace { + +void print_input(const Real t, const Real p, const Real wsec) { + std::cout << "T_mid:" << t << std::endl; + std::cout << "p_mid:" << p << std::endl; + std::cout << "wsec:" << wsec << std::endl; +} + +void print_output(const Real w0, const Real rho, const Real tke, + const Real wsub, const Real wice, const Real wsig, + const Real naai_hom, const Real naai, const Real rpdel, + MAMAci::view_3d factnum, const Real tendnd, + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + const Real hetfrz_immersion_nucleation_tend, + const Real hetfrz_contact_nucleation_tend, + const Real hetfrz_depostion_nucleation_tend, const int kb) { + std::cout << "w0:" << w0 << std::endl; + std::cout << " rho: " << rho << std::endl; + std::cout << "TKE:" << tke << std::endl; + std::cout << "WSUB:" << wsub << std::endl; + std::cout << "WICE:" << wice << std::endl; + std::cout << "WSIG:" << wsig << std::endl; + std::cout << "naai_hom_:" << naai_hom << std::endl; + std::cout << "naai_:" << naai << std::endl; + std::cout << "rpdel_:" << rpdel << std::endl; + std::cout << "factnum_:" << factnum(0, 0, kb) << " : " << factnum(0, 1, kb) + << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) + << std::endl; + std::cout << "tendnd_:" << tendnd << std::endl; + for(int ic = 9; ic < 40; ++ic) { + std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; + } + for(int ic = 0; ic < 25; ++ic) { + std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) + << std::endl; + } + + std::cout << "hetfrz_immersion_nucleation_tend_:" + << hetfrz_immersion_nucleation_tend << ":" + << hetfrz_immersion_nucleation_tend << std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" + << hetfrz_contact_nucleation_tend << std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" + << hetfrz_depostion_nucleation_tend << std::endl; +} + +void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, + const int ncol_, const int nlev_) { + const Real w_sec_e3sm[73] = { + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.44201892319518146E-003, + 0.77315620137962326E-003, 0.24806301482800117E-002, + 0.11109772692898754E-001, 0.40651094032490273E-001, + 0.82156694426095800E-001, 0.12207124453993526E+000, + 0.15516728994634110E+000, 0.17775318086169636E+000, + 0.18549817250146838E+000, 0.17184548286554119E+000, + 0.12741230682196053E+000, 0.65495229516041628E-001, + 0.26909155217660592E-001}; + + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + w_sec_int_(icol, kk) = w_sec_e3sm[kk]; + // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + } + // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; + } + const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, + 0.75991761081225006E-002, 0.12291092068185365E-001, + 0.11484807652762415E-001, 0.10856880396302943E-001, + 0.10500384508819637E-001, 0.10361486171738229E-001, + 0.10333325067964508E-001, 0.10258838031435397E-001, + 0.10027325248446619E-001, 0.97784259072973521E-002, + 0.96611845055866539E-002, 0.96249746122327937E-002, + 0.95773431515696512E-002, 0.95180614513688099E-002, + 0.94713233348487150E-002, 0.94503864489758338E-002, + 0.94536294366578833E-002, 0.94575972194308883E-002, + 0.94403767489615684E-002, 0.93975694769176284E-002, + 0.93322843554751022E-002, 0.92777070192527501E-002, + 0.92456776697171228E-002, 0.92266924824142716E-002, + 0.92123025773060436E-002, 0.91888715633294191E-002, + 0.91516797753615851E-002, 0.90958299606649744E-002, + 0.89988037524983237E-002, 0.88220984587642423E-002, + 0.85231270833157156E-002, 0.81397522619395188E-002, + 0.79160421807845088E-002, 0.81206851117902653E-002, + 0.86526891616674779E-002, 0.91682975412125615E-002, + 0.96043394254592580E-002, 0.10033689085881327E-001, + 0.10428656694074272E-001, 0.10715913043864789E-001, + 0.10919631245454951E-001, 0.11250937075285789E-001, + 0.11829292157343831E-001, 0.12413311776454055E-001, + 0.12851317662157077E-001, 0.13175523677700330E-001, + 0.13224182907540188E-001, 0.13085937680733115E-001, + 0.12615055546741534E-001, 0.11995423733019836E-001, + 0.12346556881757400E-001, 0.13433752971524651E-001, + 0.13904308240950175E-001, 0.13539811748121957E-001, + 0.12555099320041433E-001, 0.11519643673351362E-001, + 0.11414071302852231E-001, 0.13409756835238139E-001, + 0.24071962815959351E-001, 0.75489419450816414E-001, + 0.62082011878960308E+000, 0.63952862312816796E+001, + 0.16226857944175123E+002, 0.21882852534279891E+002, + 0.24966173574402408E+002, 0.25710753126453692E+002, + 0.24069881024271943E+002, 0.19743922403487922E+002, + 0.98667814246712027E+001, 0.25633359450143991E+001, + 0.14682471685037493E+001}; + // compute eddy diffusivity of heat at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + // kvh_int_(icol, kk) = kvh_mid_(icol, kk); + kvh_int_(icol, kk) = kvh_e3sm[kk]; + } + // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; + } +} +void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, + const int nlev_) { + const Real dgnum_ait_e3sm[72] = { + 0.20877713336487552E-007, 0.21782230353342090E-007, + 0.21688324003865861E-007, 0.21112855042342451E-007, + 0.19162058462939536E-007, 0.18102979880838476E-007, + 0.17906980715477606E-007, 0.20271254074583327E-007, + 0.22698983422181942E-007, 0.24134835117044986E-007, + 0.25498156808001372E-007, 0.29796738799905547E-007, + 0.35822987394021908E-007, 0.41170963764365215E-007, + 0.44892726528330642E-007, 0.47217231342203107E-007, + 0.48928661807108766E-007, 0.50170939816128735E-007, + 0.51078750853732200E-007, 0.52247333465736065E-007, + 0.53190758580174931E-007, 0.53576491941850044E-007, + 0.53915614473890715E-007, 0.54510964775236826E-007, + 0.55643231691556703E-007, 0.57057811112589899E-007, + 0.58177383586181116E-007, 0.58209849180850108E-007, + 0.57976751598840998E-007, 0.52000000000000002E-007, + 0.50728746567226150E-007, 0.49119902704480870E-007, + 0.48212162162050883E-007, 0.49227715213506454E-007, + 0.46876827233752246E-007, 0.45360603896257791E-007, + 0.49986783979004747E-007, 0.51186879246229022E-007, + 0.50009353247048599E-007, 0.48250264542204811E-007, + 0.47560278748093609E-007, 0.48298089720730957E-007, + 0.49095935613468768E-007, 0.49493024126912931E-007, + 0.50250797590476007E-007, 0.51949267668322422E-007, + 0.53778727208416418E-007, 0.53563593301099588E-007, + 0.51218136771199298E-007, 0.43171429694325200E-007, + 0.39019610039033895E-007, 0.36175109143257051E-007, + 0.42731638777892750E-007, 0.38060728507221777E-007, + 0.44046323901481340E-007, 0.39216732751330010E-007, + 0.34842233953609988E-007, 0.34068804733226066E-007, + 0.30636043694263528E-007, 0.28302341686131413E-007, + 0.33023014309036320E-007, 0.34745748365385196E-007, + 0.43623545003583371E-007, 0.48206451795644064E-007, + 0.49854490325455530E-007, 0.50346335647724146E-007, + 0.50661560988561763E-007, 0.50986261962838767E-007, + 0.51256955985111086E-007, 0.51482578449096488E-007, + 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + } + } +} + KOKKOS_INLINE_FUNCTION void copy_scream_array_to_mam4xx(const haero::ThreadTeam &team, const MAMAci::view_2d mam4xx_view, @@ -1150,10 +1338,11 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - // for (int kk=0; kk<72; ++kk){ - // std::cout << "num_cb_accum_top:"<::get_thread_range_parallel_scan_team_policy(ncol_, // nlev_); @@ -1167,127 +1356,29 @@ void MAMAci::run_impl(const double dt) { // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); - const int kb = 62; compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); - // std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; - // std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; - // std::cout << "wsec:" << w_sec_(0, kb) << std::endl; - // std::cout << "w0:" << w0_(0, kb) << std::endl; - // std::cout << " rho: " << rho_(0, kb) << std::endl; - - // compute vertical velocity variance at the interfaces - const Real w_sec_e3sm[73] = { - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.44201892319518146E-003, - 0.77315620137962326E-003, 0.24806301482800117E-002, - 0.11109772692898754E-001, 0.40651094032490273E-001, - 0.82156694426095800E-001, 0.12207124453993526E+000, - 0.15516728994634110E+000, 0.17775318086169636E+000, - 0.18549817250146838E+000, 0.17184548286554119E+000, - 0.12741230682196053E+000, 0.65495229516041628E-001, - 0.26909155217660592E-001}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_e3sm[kk]; - // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); - } - // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); - w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; - } compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output tke_); - // std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. compute_subgrid_scale_velocities(team_policy, tke_, wsubmin_, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); - // std::cout << "WSUB:" << wsub_(0, kb) << std::endl; - // std::cout << "WICE:" << wsubice_(0, kb) << std::endl; - // std::cout << "WSIG:" << wsig_(0, kb) << std::endl; Kokkos::fence(); // wait for wsig_ to be computed. compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - const Real dgnum_ait_e3sm[72] = { - 0.20877713336487552E-007, 0.21782230353342090E-007, - 0.21688324003865861E-007, 0.21112855042342451E-007, - 0.19162058462939536E-007, 0.18102979880838476E-007, - 0.17906980715477606E-007, 0.20271254074583327E-007, - 0.22698983422181942E-007, 0.24134835117044986E-007, - 0.25498156808001372E-007, 0.29796738799905547E-007, - 0.35822987394021908E-007, 0.41170963764365215E-007, - 0.44892726528330642E-007, 0.47217231342203107E-007, - 0.48928661807108766E-007, 0.50170939816128735E-007, - 0.51078750853732200E-007, 0.52247333465736065E-007, - 0.53190758580174931E-007, 0.53576491941850044E-007, - 0.53915614473890715E-007, 0.54510964775236826E-007, - 0.55643231691556703E-007, 0.57057811112589899E-007, - 0.58177383586181116E-007, 0.58209849180850108E-007, - 0.57976751598840998E-007, 0.52000000000000002E-007, - 0.50728746567226150E-007, 0.49119902704480870E-007, - 0.48212162162050883E-007, 0.49227715213506454E-007, - 0.46876827233752246E-007, 0.45360603896257791E-007, - 0.49986783979004747E-007, 0.51186879246229022E-007, - 0.50009353247048599E-007, 0.48250264542204811E-007, - 0.47560278748093609E-007, 0.48298089720730957E-007, - 0.49095935613468768E-007, 0.49493024126912931E-007, - 0.50250797590476007E-007, 0.51949267668322422E-007, - 0.53778727208416418E-007, 0.53563593301099588E-007, - 0.51218136771199298E-007, 0.43171429694325200E-007, - 0.39019610039033895E-007, 0.36175109143257051E-007, - 0.42731638777892750E-007, 0.38060728507221777E-007, - 0.44046323901481340E-007, 0.39216732751330010E-007, - 0.34842233953609988E-007, 0.34068804733226066E-007, - 0.30636043694263528E-007, 0.28302341686131413E-007, - 0.33023014309036320E-007, 0.34745748365385196E-007, - 0.43623545003583371E-007, 0.48206451795644064E-007, - 0.49854490325455530E-007, 0.50346335647724146E-007, - 0.50661560988561763E-007, 0.50986261962838767E-007, - 0.51256955985111086E-007, 0.51482578449096488E-007, - 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - } - } + + set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - // std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; + // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is // equivalent to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct @@ -1297,91 +1388,17 @@ void MAMAci::run_impl(const double dt) { // output nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - naai_(icol, kk) = 9876.0; - } - } - store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, // output cloud_frac_, cloud_frac_prev_); - // std::cout << "naai_hom_:" << naai_hom_(0, kb) << std::endl; - // std::cout << "naai_:" << naai_(0, kb) << std::endl; - - // MUST FIXME: save cloud borne aerosols here!!!! - //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous - // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne - // aerosols, not all the cloud borne aerosols. - //------------------------------------------------------------- - /*NOTE: We probably need to store indices for the select few cloud borne - aerosols Fortran code: lchnk_zb = lchnk - begchunk ! save copy of cloud borne - aerosols for use in heterogeneous freezing before ! we change it in dropmixnuc - do ispec = 1, ncnst - call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) - aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) - aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * - rho(:ncol,:) enddo - */ - compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); - // std::cout << "rpdel_:" << rpdel_(0, kb) << std::endl; Kokkos::fence(); // wait for rpdel_ to be computed. - const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, - 0.75991761081225006E-002, 0.12291092068185365E-001, - 0.11484807652762415E-001, 0.10856880396302943E-001, - 0.10500384508819637E-001, 0.10361486171738229E-001, - 0.10333325067964508E-001, 0.10258838031435397E-001, - 0.10027325248446619E-001, 0.97784259072973521E-002, - 0.96611845055866539E-002, 0.96249746122327937E-002, - 0.95773431515696512E-002, 0.95180614513688099E-002, - 0.94713233348487150E-002, 0.94503864489758338E-002, - 0.94536294366578833E-002, 0.94575972194308883E-002, - 0.94403767489615684E-002, 0.93975694769176284E-002, - 0.93322843554751022E-002, 0.92777070192527501E-002, - 0.92456776697171228E-002, 0.92266924824142716E-002, - 0.92123025773060436E-002, 0.91888715633294191E-002, - 0.91516797753615851E-002, 0.90958299606649744E-002, - 0.89988037524983237E-002, 0.88220984587642423E-002, - 0.85231270833157156E-002, 0.81397522619395188E-002, - 0.79160421807845088E-002, 0.81206851117902653E-002, - 0.86526891616674779E-002, 0.91682975412125615E-002, - 0.96043394254592580E-002, 0.10033689085881327E-001, - 0.10428656694074272E-001, 0.10715913043864789E-001, - 0.10919631245454951E-001, 0.11250937075285789E-001, - 0.11829292157343831E-001, 0.12413311776454055E-001, - 0.12851317662157077E-001, 0.13175523677700330E-001, - 0.13224182907540188E-001, 0.13085937680733115E-001, - 0.12615055546741534E-001, 0.11995423733019836E-001, - 0.12346556881757400E-001, 0.13433752971524651E-001, - 0.13904308240950175E-001, 0.13539811748121957E-001, - 0.12555099320041433E-001, 0.11519643673351362E-001, - 0.11414071302852231E-001, 0.13409756835238139E-001, - 0.24071962815959351E-001, 0.75489419450816414E-001, - 0.62082011878960308E+000, 0.63952862312816796E+001, - 0.16226857944175123E+002, 0.21882852534279891E+002, - 0.24966173574402408E+002, 0.25710753126453692E+002, - 0.24069881024271943E+002, 0.19743922403487922E+002, - 0.98667814246712027E+001, 0.25633359450143991E+001, - 0.14682471685037493E+001}; - // compute eddy diffusivity of heat at the interfaces - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - // kvh_int_(icol, kk) = kvh_mid_(icol, kk); - kvh_int_(icol, kk) = kvh_e3sm[kk]; - } - // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); - kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; - } - call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output @@ -1393,17 +1410,13 @@ void MAMAci::run_impl(const double dt) { dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - // std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, - // kb)<<" : "<( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1419,20 +1432,19 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); + + print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), + wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), + rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, + qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), + hetfrz_contact_nucleation_tend_(0, kb), + hetfrz_depostion_nucleation_tend_(0, kb), kb); const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); if(ans < 5.65184e-06 || ans > 5.65186e-06) { - std::cout << "SOmethign changed!!!! :" + std::cout << "Somethign changed!!!! :" << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; exit(1); } - std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend_(0, kb) << ":" - << hetfrz_immersion_nucleation_tend_(0, 0) << std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" - << hetfrz_contact_nucleation_tend_(0, kb) << std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" - << hetfrz_depostion_nucleation_tend_(0, kb) << std::endl; - Kokkos::fence(); // wait before returning to calling function } From 831ffbbcdac29db7832d59e35c4b1d881d8752ae Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 20:51:50 -0700 Subject: [PATCH 085/142] further cleanup for the output to be used by other processes --- .../mam/eamxx_mam_aci_process_interface.cpp | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 67184b7466d..0a28c55a2e6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -502,7 +502,13 @@ void call_function_dropmixnuc( const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, - // ## outputs ## + // Following outputs are all diagnostics + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, + MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, + MAMAci::view_2d wtke, MAMAci::view_3d ccn, + + // ## outputs to be used by other processes ## // qqcw_fld_work should be directly assigned to the cloud borne aerosols MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], @@ -515,12 +521,6 @@ void call_function_dropmixnuc( // tendnd is used by microphysics scheme (e.g. P3) MAMAci::view_2d tendnd, - // Following outputs are all diagnostics - MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, - MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, - MAMAci::view_2d wtke, MAMAci::view_3d ccn, - // ## work arrays ## MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, @@ -1338,9 +1338,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - + // FIXME: Remove set_input and print_input const int kb = 62; + set_input(w_sec_int_, kvh_int_, ncol_, nlev_); print_input(dry_atm_.T_mid(0, kb), dry_atm_.p_mid(0, kb), w_sec_int_(0, kb)); const auto scan_policy = ekat::ExeSpaceUtils< @@ -1371,10 +1371,11 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for wsig_ to be computed. + // We need dry diameter for only aitken mode compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - + // FIXME:Remove set_dgait set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -1386,8 +1387,11 @@ void MAMAci::run_impl(const double dt) { nucleate_ice_, team_policy, dry_atm_, dry_aero_, wsubice_, aitken_dry_dia_, nlev_, dt, // output - nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); + nihf_, niim_, nidep_, nimey_, naai_hom_, + // ## output to be used by the other processes ## + naai_); + // Compute cloud fractions based on cloud threshold store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, // output @@ -1402,9 +1406,10 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output - qqcw_fld_work_, ptend_q_, factnum_, tendnd_, coltend_, coltend_cw_, qcld_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, + // ## output to be used by the other processes ## + qqcw_fld_work_, ptend_q_, factnum_, tendnd_, // work arrays raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); @@ -1427,7 +1432,7 @@ void MAMAci::run_impl(const double dt) { call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, - // output + // ## output to be used by the other processes ## hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, // work arrays From e7730c707ed2dbba86ca9de42bf7722b0c8cb0fd Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 21:13:54 -0700 Subject: [PATCH 086/142] Adds some comments --- .../mam/eamxx_mam_aci_process_interface.cpp | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 0a28c55a2e6..645d520ca5c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -476,10 +476,12 @@ void compute_recipical_pseudo_density(const haero::ThreadTeam &team, const int icol, const int nlev, // output MAMAci::view_2d rpdel) { - // FIXME: Add an assert to ensure pdel is non-zero Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), - KOKKOS_LAMBDA(int kk) { rpdel(icol, kk) = 1 / pdel(icol, kk); }); + Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); } void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, MAMAci::const_view_2d pdel, @@ -1403,6 +1405,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. + // Compute activated CCN number tendency (tendnd_) and updated + // cloud borne aerosols (stored in a work array) and interstitial + // aerosols tendencies call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output @@ -1415,21 +1420,13 @@ void MAMAci::run_impl(const double dt) { dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - // MUST FIXME: save cloud borne aerosols here!!!! - //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous - // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne - // aerosols, not all the cloud borne aerosols. - //------------------------------------------------------------- - - copy_mam4xx_array_to_scream( - team_policy, ptend_q_output_, ptend_q_, nlev_); - copy_mam4xx_array_to_scream( - team_policy, coltend_outp_, coltend_, nlev_); - copy_mam4xx_array_to_scream( - team_policy, coltend_cw_outp_, coltend_cw_, nlev_); + //--------------------------------------------------------------------------- + // NOTE: DO NOT UPDATE cloud borne aerosols using the qqcw_fld_work_ array + // at this point as heterozenous freezing needs to use cloud borne aerosols + // before they are changed by the droplet activation (dropmixnuc) process. + //--------------------------------------------------------------------------- + // Compute hetrozenous freezing call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, // ## output to be used by the other processes ## @@ -1438,6 +1435,7 @@ void MAMAci::run_impl(const double dt) { // work arrays diagnostic_scratch_); + // FIXME: Remove the following print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, From 7f0cf90a087ce450949c05dbfd724eef9ed66a17 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 21 Apr 2024 22:08:37 -0700 Subject: [PATCH 087/142] Adds functions to update to qqcw and state --- .../mam/eamxx_mam_aci_process_interface.cpp | 180 ++++++++++++++---- .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- 2 files changed, 148 insertions(+), 34 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 645d520ca5c..0c7c7c72d65 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -9,7 +9,18 @@ void print_input(const Real t, const Real p, const Real wsec) { std::cout << "p_mid:" << p << std::endl; std::cout << "wsec:" << wsec << std::endl; } - +void print_bef_ndrop(const mam_coupling::AerosolState &dry_aero, const int kb) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "Bef-ndrop-cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) + << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "Bef-ndrop-cldbrn-mmr:" + << dry_aero.cld_aero_mmr[m][a](0, kb) << std::endl; + } + } + } +} void print_output(const Real w0, const Real rho, const Real tke, const Real wsub, const Real wice, const Real wsig, const Real naai_hom, const Real naai, const Real rpdel, @@ -18,7 +29,8 @@ void print_output(const Real w0, const Real rho, const Real tke, MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const Real hetfrz_immersion_nucleation_tend, const Real hetfrz_contact_nucleation_tend, - const Real hetfrz_depostion_nucleation_tend, const int kb) { + const Real hetfrz_depostion_nucleation_tend, + const mam_coupling::AerosolState &dry_aero, const int kb) { std::cout << "w0:" << w0 << std::endl; std::cout << " rho: " << rho << std::endl; std::cout << "TKE:" << tke << std::endl; @@ -39,6 +51,25 @@ void print_output(const Real w0, const Real rho, const Real tke, std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) << std::endl; } + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } std::cout << "hetfrz_immersion_nucleation_tend_:" << hetfrz_immersion_nucleation_tend << ":" @@ -358,9 +389,8 @@ void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, void compute_nucleate_ice_tendencies( const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, const mam_coupling::DryAtmosphere &dry_atmosphere, - const mam_coupling::AerosolState &dry_aerosol_state, - const MAMAci::view_2d wsubice, const MAMAci::view_2d aitken_dry_dia, - const int nlev, const double dt, + const mam_coupling::AerosolState &dry_aero, const MAMAci::view_2d wsubice, + const MAMAci::view_2d aitken_dry_dia, const int nlev, const double dt, // output MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai) { @@ -385,7 +415,7 @@ void compute_nucleate_ice_tendencies( // Store interstitial and cld borne aerosols in "progrs" struture mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + mam_coupling::aerosols_for_column(dry_aero, icol); // Store atmopsheric vars (Tmid, Pmid, cloud fraction, qv, wsubmin) haero::Atmosphere haero_atm = @@ -502,7 +532,7 @@ void call_function_dropmixnuc( mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, - const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, + const mam_coupling::AerosolState &dry_aero, const int nlev, // Following outputs are all diagnostics MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], @@ -649,32 +679,46 @@ void call_function_dropmixnuc( // and Atmosphere (water species such as qv, qc etc.) // get prognostics - mam4::Prognostics progs_at_col = - aerosols_for_column(dry_aerosol_state, icol); + mam4::Prognostics progs_at_col = aerosols_for_column(dry_aero, icol); // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays + // FIXME:: Kookos for here?? for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; - // get state_q at a gri cell (col,lev) + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); - // get qqcw at a gri cell (col,lev) - mam4::utils::extract_qqcw_from_prognostics(progs_at_col, - qqcw_at_lev_col, klev); + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } - // create colum views of state_q and qqcw - for(int icnst = 15; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_work_loc(icol, klev, icnst) = - state_q_at_lev_col[icnst]; // FIXME: ensure that indices are - // right! remove "15" if possible!! - qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; + // get qqcw at a grid cell (col,lev) + // NOTE: The layout for qqcw array is based on mam_idx in dropmixnuc + // To mimic that, we are using the following for-loops + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + qqcw_view[ind_qqcw](klev) = dry_aero.cld_aero_nmr[m](icol, klev); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + qqcw_view[ind_qqcw](klev) = + dry_aero.cld_aero_mmr[m][a](icol, klev); + ++ind_qqcw; + } + } } } @@ -739,6 +783,64 @@ void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, }); } +// Update cloud borne aerosols +void update_cloud_borne_aerosols( + haero::ThreadTeamPolicy team_policy, + const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, + // output + mam_coupling::AerosolState &dry_aero) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + dry_aero.cld_aero_nmr[m](icol, kk) = + qqcw_fld_work[ind_qqcw](icol, kk); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + dry_aero.cld_aero_mmr[m][a](icol, kk) = + qqcw_fld_work[ind_qqcw](icol, kk); + ++ind_qqcw; + } + } + } + }); + }); +} + +// Update interstitial aerosols using tendencies +void update_interstitial_aerosols( + haero::ThreadTeamPolicy team_policy, + const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], const int nlev, + const Real dt, + // output + mam_coupling::AerosolState &dry_aero) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + ; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + dry_aero.int_aero_mmr[m][a](icol, kk) += + ptend_q[s_idx](icol, kk) * dt; + s_idx++; + } + } + dry_aero.int_aero_nmr[m](icol, kk) += + ptend_q[s_idx](icol, kk) * dt; + s_idx++; + } + }); + }); +} + void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, mam_coupling::DryAtmosphere &dry_atm_, @@ -749,9 +851,9 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d hetfrz_contact_nucleation_tend, MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d diagnostic_scratch_[]) { - mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState dry_aerosol_state = dry_aero_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState dry_aero = dry_aero_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; @@ -766,7 +868,7 @@ void call_hetfrz_compute_tendencies( atmosphere_for_column(dry_atmosphere, icol); haero::Surface surf{}; mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + mam_coupling::aerosols_for_column(dry_aero, icol); const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); @@ -1343,7 +1445,8 @@ void MAMAci::run_impl(const double dt) { // FIXME: Remove set_input and print_input const int kb = 62; set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - print_input(dry_atm_.T_mid(0, kb), dry_atm_.p_mid(0, kb), w_sec_int_(0, kb)); + // print_input(dry_atm_.T_mid(0, kb), dry_atm_.p_mid(0, kb), w_sec_int_(0, + // kb)); const auto scan_policy = ekat::ExeSpaceUtils< // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, @@ -1404,10 +1507,10 @@ void MAMAci::run_impl(const double dt) { rpdel_); Kokkos::fence(); // wait for rpdel_ to be computed. - - // Compute activated CCN number tendency (tendnd_) and updated - // cloud borne aerosols (stored in a work array) and interstitial - // aerosols tendencies + // print_bef_ndrop(dry_aero_, kb); + // Compute activated CCN number tendency (tendnd_) and updated + // cloud borne aerosols (stored in a work array) and interstitial + // aerosols tendencies call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output @@ -1435,13 +1538,28 @@ void MAMAci::run_impl(const double dt) { // work arrays diagnostic_scratch_); + //--------------------------------------------------------------- + //----------------- End of all processes ------------------------ + //--------------------------------------------------------------- + + // Update cloud borne aerosols + update_cloud_borne_aerosols(team_policy, qqcw_fld_work_, nlev_, + // output + dry_aero_); + + // Update interstitial aerosols using tendencies + update_interstitial_aerosols(team_policy, ptend_q_, nlev_, dt, + // output + dry_aero_); + // FIXME: Remove the following print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), hetfrz_contact_nucleation_tend_(0, kb), - hetfrz_depostion_nucleation_tend_(0, kb), kb); + hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); + const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); if(ans < 5.65184e-06 || ans > 5.65186e-06) { std::cout << "Somethign changed!!!! :" @@ -1451,10 +1569,6 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait before returning to calling function } -void MAMAci::finalize_impl() { - m_atm_logger->log(ekat::logger::LogLevel::info, "calling ACI final"); -} - } // namespace scream /* diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 297078c8727..471cec42ce6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -171,7 +171,7 @@ class MAMAci final : public scream::AtmosphereProcess { // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; - void finalize_impl() override; + void finalize_impl() const {/*DO NOTHING*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. From 19606b8c7a0f45f8e3618170534fc2f64624dcde Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 21 Apr 2024 22:45:55 -0700 Subject: [PATCH 088/142] Removes finanlize from cpp and mam_copy functions --- .../mam/eamxx_mam_aci_process_interface.cpp | 139 +----------------- .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- 2 files changed, 3 insertions(+), 138 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 0c7c7c72d65..00064e0a508 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -222,37 +222,6 @@ void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, } } -KOKKOS_INLINE_FUNCTION -void copy_scream_array_to_mam4xx(const haero::ThreadTeam &team, - const MAMAci::view_2d mam4xx_view, - MAMAci::const_view_3d scream_view, - const int icol, const int nlev, - const int view_num) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - mam4xx_view(icol, kk) = scream_view(icol, kk, view_num); - }); -} - -void copy_scream_array_to_mam4xx( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d mam4xx_views[mam4::ndrop::ncnst_tot], - MAMAci::const_view_3d scream_view, const int nlev) { - // Localize the input arrays. - MAMAci::view_2d mam4xx[mam4::ndrop::ncnst_tot]; - for(int view_num = 0; view_num < mam4::ndrop::ncnst_tot; ++view_num) - mam4xx[view_num] = mam4xx_views[view_num]; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - for(int view_num = 0; view_num < mam4::ndrop::ncnst_tot; ++view_num) { - MAMAci::view_2d mam4xx_view = mam4xx[view_num]; - copy_scream_array_to_mam4xx(team, mam4xx_view, scream_view, icol, - nlev, view_num); - } - }); -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -300,9 +269,6 @@ void compute_tke_using_w_sec(const haero::ThreadTeam &team, const int nlev, // output MAMAci::view_2d tke) { - // FIXME Is this the correct boundary condition for tke at the surface? - // TKE seems to be at interfaces but w_sec is at cell centers so this - // descrepensy needs to be worked out. Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, nlev + 1), KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); @@ -1539,7 +1505,7 @@ void MAMAci::run_impl(const double dt) { diagnostic_scratch_); //--------------------------------------------------------------- - //----------------- End of all processes ------------------------ + // Now update interstitial and cllud borne aerosols //--------------------------------------------------------------- // Update cloud borne aerosols @@ -1569,105 +1535,4 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait before returning to calling function } -} // namespace scream - -/* -FORTRAN OUTPUT: - - ----- OUPUT AT time step: 6 6 - wo 3.343177351577457E-004 - rho: 1.28186433384113 - TKE: 1.666465903934813E-002 6.097664104873541E-002 - wsub: 0.160873967324407 - wsubi: 0.200000000000000 - wsig: 0.160873967324407 - dgnum_ait: 4.362354500358337E-008 - naai: 11464.5957222634 - naai_hom: 29838.6879763933 - aer_cb: so4_c1 4.327927689273498E-012 so4_c1 4.327927689273498E-012 - aer_cb: bc_c1 6.811694223739704E-014 bc_c1 6.811694223739704E-014 - aer_cb: pom_c1 8.647238075040541E-013 pom_c1 8.647238075040541E-013 - aer_cb: soa_c1 6.734058317716400E-012 soa_c1 6.734058317716400E-012 - aer_cb: dst_c1 8.239303776951524E-014 dst_c1 8.239303776951524E-014 - aer_cb: ncl_c1 2.685810991448475E-014 ncl_c1 2.685810991448475E-014 - aer_cb: mom_c1 2.218253512777307E-015 mom_c1 2.218253512777307E-015 - aer_cb: num_c1 477478.884677930 num_c1 477478.884677930 - aer_cb: dst_c3 2.869197497896834E-012 dst_c3 2.869197497896834E-012 - aer_cb: ncl_c3 4.636857908110365E-013 ncl_c3 4.636857908110365E-013 - aer_cb: so4_c3 4.375223642985256E-014 so4_c3 4.375223642985256E-014 - aer_cb: bc_c3 3.674860666928688E-016 bc_c3 3.674860666928688E-016 - aer_cb: pom_c3 1.114802999982066E-015 pom_c3 1.114802999982066E-015 - aer_cb: soa_c3 8.915776993183282E-015 soa_c3 8.915776993183282E-015 - aer_cb: mom_c3 7.802642470782658E-017 mom_c3 7.802642470782658E-017 - aer_cb: num_c3 151.593971049442 num_c3 151.593971049442 - aer_cb: bc_c4 1.127495264613331E-027 bc_c4 1.127495264613331E-027 - aer_cb: pom_c4 8.204883715624573E-027 pom_c4 8.204883715624573E-027 - aer_cb: mom_c4 3.249069782816687E-033 mom_c4 3.249069782816687E-033 - aer_cb: num_c4 1.345885758586041E-013 num_c4 1.345885758586041E-013 - tendnd: -839.404918218856 7289315.36108503 7792958.31201634 - 1.666666666666667E-003 - factnum: 0.998556176544263 0.861689826773470 - 0.999999999974140 0.000000000000000E+000 - nctend_mixnuc: -839.404918218856 - O3_tend: 0.000000000000000E+000 - H2O2_tend: 0.000000000000000E+000 - H2SO4_tend: 0.000000000000000E+000 - SO2_tend: 0.000000000000000E+000 - DMS_tend: 0.000000000000000E+000 - SOAG_tend: 0.000000000000000E+000 - so4_a1_tend: -1.384310943569190E-014 - pom_a1_tend: -2.950799002682271E-015 - soa_a1_tend: -2.283869371271150E-014 - bc_a1_tend: -2.202672700280516E-016 - dst_a1_tend: -2.194060414083922E-016 - ncl_a1_tend: -6.981124745267083E-017 - mom_a1_tend: -5.662690535048673E-018 - num_a1_tend: -1626.38730532537 - so4_a2_tend: -2.548707408341951E-017 - soa_a2_tend: -1.929990276139139E-018 - ncl_a2_tend: -6.855087738119723E-019 - mom_a2_tend: -4.571691604185304E-020 - num_a2_tend: -111.890330245159 - dst_a3_tend: -7.609611333715698E-015 - ncl_a3_tend: -1.256350252705383E-015 - so4_a3_tend: -1.320988655880598E-016 - bc_a3_tend: -1.099586316024402E-018 - pom_a3_tend: -3.309939677824050E-018 - soa_a3_tend: -2.767668335981830E-017 - mom_a3_tend: -2.254911604324925E-019 - num_a3_tend: -0.406788905791186 - pom_a4_tend: -2.375070578317096E-017 - bc_a4_tend: 4.082479713528180E-017 - mom_a4_tend: -4.901356204764327E-023 - num_a4_tend: -34.4232851017138 - so4_c1: 1322558.40616450 - pom_c1: 1.233997159684093E-011 - soa_c1: 2.393397505793018E-012 - bc_c1: 1.872520079842212E-011 - dst_c1: 1.922157644491776E-013 - ncl_c1: 2.530385880698310E-013 - mom_c1: 8.315014074126540E-014 - num_c1: 6.895143982427237E-015 - so4_c2: 142214.467855250 - soa_c2: 3.181118913367564E-014 - ncl_c2: 2.996638282752921E-015 - mom_c2: 8.055333842809433E-016 - num_c2: 5.202632275847699E-017 - dst_c3: 464.524744048298 - ncl_c3: 8.814810290885256E-012 - so4_c3: 1.417415096039318E-012 - bc_c3: 1.284022582681906E-013 - pom_c3: 1.081226492056768E-015 - soa_c3: 3.292708392236485E-015 - mom_c3: 2.584587804061707E-014 - num_c3: 2.324887367507579E-016 - pom_c4: 6.934165370527951E-014 - bc_c4: 2.703924752644086E-026 - mom_c4: 3.684880329580097E-027 - num_c4: 1.049556651713384E-032 - - frzimm: 5.651860156917002E-006 - frzcnt: 0.000000000000000E+000 - frzdep: 0.000000000000000E+000 - - */ \ No newline at end of file +} // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 471cec42ce6..1fa255ccdf7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -171,7 +171,7 @@ class MAMAci final : public scream::AtmosphereProcess { // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; - void finalize_impl() const {/*DO NOTHING*/}; + void finalize_impl(){/*DO NOTHING*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. From 9f8b2af8c1295c0e281c4623ff84c79dd5349732 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 21 Apr 2024 23:01:59 -0700 Subject: [PATCH 089/142] Removes another set of unused mam copy functions --- .../mam/eamxx_mam_aci_process_interface.cpp | 49 ++++--------------- 1 file changed, 9 insertions(+), 40 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 00064e0a508..da0632fd619 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -359,7 +359,9 @@ void compute_nucleate_ice_tendencies( const MAMAci::view_2d aitken_dry_dia, const int nlev, const double dt, // output MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, - MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai) { + MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, + // ## output used by other processes ## + MAMAci::view_2d naai) { //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation @@ -368,9 +370,6 @@ void compute_nucleate_ice_tendencies( Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // for(int icol = 0; icol < 5; ++icol) { - // std::cout<<""< -void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, - MAMAci::view_3d scream, - MAMAci::view_2d mam4xx[len], const int nlev) { - // Localize the input arrays. - MAMAci::view_2d mam4xx_loc[len]; - for(int view_num = 0; view_num < len; ++view_num) - mam4xx_loc[view_num] = mam4xx[view_num]; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - for(int view_num = 0; view_num < len; ++view_num) { - MAMAci::view_2d mam4xx_view = mam4xx_loc[view_num]; - copy_mam4xx_array_to_scream(team, scream, mam4xx_view, icol, nlev, - view_num); - } - }); } // Update cloud borne aerosols @@ -790,7 +758,6 @@ void update_interstitial_aerosols( Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - ; for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { for(int a = 0; a < mam4::num_species_mode(m); ++a) { if(dry_aero.int_aero_mmr[m][a].data()) { @@ -909,7 +876,7 @@ void call_hetfrz_compute_tendencies( // values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; - const Real t = 0; + const Real t = 0; // not used hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, diags, tends); }); @@ -1085,6 +1052,11 @@ void MAMAci::set_grids( // ------------------------------------------------------------------------ // Output from ice nucleation process // ------------------------------------------------------------------------ + + // number of activated aerosol for ice nucleation[#/kg] + add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); + + // FIXME: Diagnostics output const auto m3_inv = 1 / m / m / m; // inverse of m3 // number conc of ice nuclei due to heterogeneous freezing [1/m3] add_field("icenuc_num_hetfrz", scalar3d_layout_mid, m3_inv, @@ -1106,9 +1078,6 @@ void MAMAci::set_grids( add_field("num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, n_unit, grid_name); - // number of activated aerosol for ice nucleation[#/kg] - add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); - // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ From 931278e07f6cbad7ce292ff13b3020f8e450e6bf Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 21 Apr 2024 23:12:23 -0700 Subject: [PATCH 090/142] Removes nihf_ from FM --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index da0632fd619..ce3ad0c0d97 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1058,9 +1058,6 @@ void MAMAci::set_grids( // FIXME: Diagnostics output const auto m3_inv = 1 / m / m / m; // inverse of m3 - // number conc of ice nuclei due to heterogeneous freezing [1/m3] - add_field("icenuc_num_hetfrz", scalar3d_layout_mid, m3_inv, - grid_name); // number conc of ice nuclei due to immersionfreezing (hetero nuc) [1/m3] add_field("icenuc_num_immfrz", scalar3d_layout_mid, m3_inv, @@ -1235,7 +1232,6 @@ void MAMAci::initialize_impl(const RunType run_type) { // computed updraft velocity dry_atm_.w_updraft = buffer_.w_updraft; - nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); niim_ = get_field_out("icenuc_num_immfrz").get_view(); nidep_ = get_field_out("icenuc_num_depnuc").get_view(); nimey_ = get_field_out("icenuc_num_meydep").get_view(); @@ -1306,6 +1302,10 @@ void MAMAci::initialize_impl(const RunType run_type) { // Allocate memory for the class members // (Kokkos::resize only works on host to allocates memory) + + // number conc of ice nuclei due to heterogeneous freezing [1/m3] + Kokkos::resize(nihf_, ncol_, nlev_); + Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); From 5f23711c718d6238c0cf4148cfc327b4c7ce3042 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 09:56:52 -0700 Subject: [PATCH 091/142] Removes all ice nuc diags and ccn from FM --- .../mam/eamxx_mam_aci_process_interface.cpp | 68 ++++++++----------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ce3ad0c0d97..8636fd1cbd0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1056,25 +1056,6 @@ void MAMAci::set_grids( // number of activated aerosol for ice nucleation[#/kg] add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); - // FIXME: Diagnostics output - const auto m3_inv = 1 / m / m / m; // inverse of m3 - - // number conc of ice nuclei due to immersionfreezing (hetero nuc) [1/m3] - add_field("icenuc_num_immfrz", scalar3d_layout_mid, m3_inv, - grid_name); - - // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] - add_field("icenuc_num_depnuc", scalar3d_layout_mid, m3_inv, - grid_name); - - // number conc of ice nuclei due to meyers deposition [1/m3] - add_field("icenuc_num_meydep", scalar3d_layout_mid, m3_inv, - grid_name); - - // number of activated aerosol for ice nucleation(homogeneous frz only)[#/kg] - add_field("num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, - n_unit, grid_name); - // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ @@ -1114,15 +1095,6 @@ void MAMAci::set_grids( // subgrid vertical velocity [m/s] add_field("wtke", scalar3d_layout_mid, m / s, grid_name); - constexpr int psat = mam4::ndrop::psat; - FieldLayout scalar4d_layout_psat_mid{{COL, LEV, MAM_PSAT}, - {ncol_, nlev_, psat}}; - // number conc of aerosols activated at supersat [#/m^3] - // NOTE: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] - // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] - add_field("ccn", scalar4d_layout_psat_mid, m3_inv, grid_name); - constexpr int num_aero_const = mam4::ndrop::ncnst_tot; FieldLayout scalar4d_layout_naero_const_mid{{COL, LEV, MAM_AERO_NCNST}, {ncol_, nlev_, num_aero_const}}; @@ -1232,12 +1204,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // computed updraft velocity dry_atm_.w_updraft = buffer_.w_updraft; - niim_ = get_field_out("icenuc_num_immfrz").get_view(); - nidep_ = get_field_out("icenuc_num_depnuc").get_view(); - nimey_ = get_field_out("icenuc_num_meydep").get_view(); - naai_hom_ = - get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); - + // ------------------------------------------------------------------------ + // Output fields to be used by other processes + // ------------------------------------------------------------------------ naai_ = get_field_out("ni_activated").get_view(); qcld_ = get_field_out("qcld").get_view(); ptend_q_output_ = get_field_out("ptend_q").get_view(); @@ -1247,7 +1216,6 @@ void MAMAci::initialize_impl(const RunType run_type) { ndropmix_ = get_field_out("ndropmix").get_view(); nsource_ = get_field_out("nsource").get_view(); wtke_ = get_field_out("wtke").get_view(); - ccn_ = get_field_out("ccn").get_view(); coltend_outp_ = get_field_out("coltend").get_view(); coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); hetfrz_immersion_nucleation_tend_ = @@ -1300,11 +1268,10 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + //--------------------------------------------------------------------------------- // Allocate memory for the class members // (Kokkos::resize only works on host to allocates memory) - - // number conc of ice nuclei due to heterogeneous freezing [1/m3] - Kokkos::resize(nihf_, ncol_, nlev_); + //--------------------------------------------------------------------------------- Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); @@ -1318,6 +1285,31 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); Kokkos::resize(rpdel_, ncol_, nlev_); + //--------------------------------------------------------------------------------- + // Diagnotics variables from the ice nucleation scheme + //--------------------------------------------------------------------------------- + + // number conc of ice nuclei due to heterogeneous freezing [1/m3] + Kokkos::resize(nihf_, ncol_, nlev_); + + // number conc of ice nuclei due to immersionfreezing (hetero nuc) [1/m3] + Kokkos::resize(niim_, ncol_, nlev_); + + // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] + Kokkos::resize(nidep_, ncol_, nlev_); + + // number conc of ice nuclei due to meyers deposition [1/m3] + Kokkos::resize(nimey_, ncol_, nlev_); + + // number of activated aerosol for ice nucleation(homogeneous frz only)[#/kg] + Kokkos::resize(naai_hom_, ncol_, nlev_); + + // number conc of aerosols activated at supersat [#/m^3] + // NOTE: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] + // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] + Kokkos::resize(ccn_, ncol_, nlev_, mam4::ndrop::psat); + for(int i = 0; i < dropmix_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } From f4b01a6cda3bad56d1a64b9965eb30e24ef3492b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 11:40:35 -0700 Subject: [PATCH 092/142] Removes some unused variables from FM --- .../mam/eamxx_mam_aci_process_interface.cpp | 48 ++++++------------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 -- 2 files changed, 14 insertions(+), 38 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 8636fd1cbd0..4577f073234 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1060,13 +1060,6 @@ void MAMAci::set_grids( // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ - constexpr int pcnst = mam4::aero_model::pcnst; - FieldLayout scalar4d_layout_nconst_mid{{COL, LEV, MAM_NCNST}, - {ncol_, nlev_, pcnst}}; - - // tendencies for interstitial and cloud borne aerosols [kg/kg or #/kg] - add_field("ptend_q", scalar4d_layout_nconst_mid, q_unit, grid_name); - // tendency in droplet number mixing ratio [#/kg/s] add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); @@ -1095,28 +1088,15 @@ void MAMAci::set_grids( // subgrid vertical velocity [m/s] add_field("wtke", scalar3d_layout_mid, m / s, grid_name); - constexpr int num_aero_const = mam4::ndrop::ncnst_tot; - FieldLayout scalar4d_layout_naero_const_mid{{COL, LEV, MAM_AERO_NCNST}, - {ncol_, nlev_, num_aero_const}}; - - // column tendency for diagnostic output - add_field("coltend", scalar4d_layout_naero_const_mid, nondim, - grid_name); - - // column tendency - add_field("coltend_cw", scalar4d_layout_naero_const_mid, nondim, - grid_name); + // ------------------------------------------------------------------------ + // Output from hetrozenous freezing + // ------------------------------------------------------------------------ const auto cm = m / 100; // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); - - // ------------------------------------------------------------------------ - // Output from hetrozenous freezing - // ------------------------------------------------------------------------ - // heterogeous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); @@ -1207,17 +1187,15 @@ void MAMAci::initialize_impl(const RunType run_type) { // ------------------------------------------------------------------------ // Output fields to be used by other processes // ------------------------------------------------------------------------ - naai_ = get_field_out("ni_activated").get_view(); - qcld_ = get_field_out("qcld").get_view(); - ptend_q_output_ = get_field_out("ptend_q").get_view(); - tendnd_ = get_field_out("nc_nuceat_tend").get_view(); - factnum_ = get_field_out("factnum").get_view(); - ndropcol_ = get_field_out("ndropcol").get_view(); - ndropmix_ = get_field_out("ndropmix").get_view(); - nsource_ = get_field_out("nsource").get_view(); - wtke_ = get_field_out("wtke").get_view(); - coltend_outp_ = get_field_out("coltend").get_view(); - coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); + naai_ = get_field_out("ni_activated").get_view(); + qcld_ = get_field_out("qcld").get_view(); + tendnd_ = get_field_out("nc_nuceat_tend").get_view(); + factnum_ = get_field_out("factnum").get_view(); + ndropcol_ = get_field_out("ndropcol").get_view(); + ndropmix_ = get_field_out("ndropmix").get_view(); + nsource_ = get_field_out("nsource").get_view(); + wtke_ = get_field_out("wtke").get_view(); + hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); hetfrz_contact_nucleation_tend_ = @@ -1317,7 +1295,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. // FIXME: Do we need these? + // column tendency for diagnostic output Kokkos::resize(coltend_[i], ncol_, nlev_); + // column tendency Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } for(int i = 0; i < mam4::aero_model::pcnst; ++i) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 1fa255ccdf7..c7198818572 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -76,9 +76,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d cloud_frac_prev_; view_2d qcld_; view_2d tendnd_; - // ptend_q_ is just ptend_q_output_ reformatted. view_2d ptend_q_[mam4::aero_model::pcnst]; - view_3d ptend_q_output_; view_3d factnum_; const_view_3d qqcw_input_; view_2d qqcw_[mam4::ndrop::ncnst_tot]; @@ -87,9 +85,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d nsource_; view_2d wtke_; view_3d ccn_; - view_3d coltend_outp_; view_2d coltend_[mam4::ndrop::ncnst_tot]; - view_3d coltend_cw_outp_; view_2d coltend_cw_[mam4::ndrop::ncnst_tot]; // raercol_cw_ and raercol_ are work arrays for dropmixnuc, allocated on the From 4d3b83dcf5dc9a36b7e4b8d81611f6de1459ef5d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 11:59:59 -0700 Subject: [PATCH 093/142] Removed all diagnostics from FM --- .../mam/eamxx_mam_aci_process_interface.cpp | 93 ++++++++++--------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4577f073234..b18984931cd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1064,30 +1064,6 @@ void MAMAci::set_grids( add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); - // activation fraction for aerosol number [fraction] - add_field("factnum", scalar4d_layout_mid, nondim, grid_name); - - // NOTE: Here is a series of internal dropmixnuc variables; - // maybe we should move them to diagnostics later (FIXME) - - // cloud droplet number mixing ratio [#/kg] - add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); - - auto inv_m2 = 1 / m / m; - inv_m2.set_string("#/m2"); - - // column-integrated droplet number [#/m2] - add_field("ndropcol", scalar3d_layout_mid, inv_m2, grid_name); - - // droplet number mixing ratio tendency due to mixing [#/kg/s] - add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); - - // droplet number mixing ratio source tendency [#/kg/s] - add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); - - // subgrid vertical velocity [m/s] - add_field("wtke", scalar3d_layout_mid, m / s, grid_name); - // ------------------------------------------------------------------------ // Output from hetrozenous freezing // ------------------------------------------------------------------------ @@ -1187,21 +1163,11 @@ void MAMAci::initialize_impl(const RunType run_type) { // ------------------------------------------------------------------------ // Output fields to be used by other processes // ------------------------------------------------------------------------ - naai_ = get_field_out("ni_activated").get_view(); - qcld_ = get_field_out("qcld").get_view(); - tendnd_ = get_field_out("nc_nuceat_tend").get_view(); - factnum_ = get_field_out("factnum").get_view(); - ndropcol_ = get_field_out("ndropcol").get_view(); - ndropmix_ = get_field_out("ndropmix").get_view(); - nsource_ = get_field_out("nsource").get_view(); - wtke_ = get_field_out("wtke").get_view(); + // ice nucleation output + naai_ = get_field_out("ni_activated").get_view(); - hetfrz_immersion_nucleation_tend_ = - get_field_out("hetfrz_immersion_nucleation_tend").get_view(); - hetfrz_contact_nucleation_tend_ = - get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = - get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + // droplet activation output + tendnd_ = get_field_out("nc_nuceat_tend").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -1246,6 +1212,14 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // hetrozenous freezing outputs + hetfrz_immersion_nucleation_tend_ = + get_field_out("hetfrz_immersion_nucleation_tend").get_view(); + hetfrz_contact_nucleation_tend_ = + get_field_out("hetfrz_contact_nucleation_tend").get_view(); + hetfrz_depostion_nucleation_tend_ = + get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + //--------------------------------------------------------------------------------- // Allocate memory for the class members // (Kokkos::resize only works on host to allocates memory) @@ -1282,19 +1256,39 @@ void MAMAci::initialize_impl(const RunType run_type) { // number of activated aerosol for ice nucleation(homogeneous frz only)[#/kg] Kokkos::resize(naai_hom_, ncol_, nlev_); + //--------------------------------------------------------------------------------- + // Diagnotics variables from the droplet activation scheme + //--------------------------------------------------------------------------------- + + // activation fraction for aerosol number [fraction] + const int num_aero_modes = mam_coupling::num_aero_modes(); + Kokkos::resize(factnum_, ncol_, num_aero_modes, nlev_); + + // cloud droplet number mixing ratio [#/kg] + Kokkos::resize(qcld_, ncol_, nlev_); + // number conc of aerosols activated at supersat [#/m^3] // NOTE: activation fraction fluxes are defined as // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] Kokkos::resize(ccn_, ncol_, nlev_, mam4::ndrop::psat); + // column-integrated droplet number [#/m2] + Kokkos::resize(ndropcol_, ncol_, nlev_); + + // droplet number mixing ratio tendency due to mixing [#/kg/s] + Kokkos::resize(ndropmix_, ncol_, nlev_); + + // droplet number mixing ratio source tendency [#/kg/s] + Kokkos::resize(nsource_, ncol_, nlev_); + + // subgrid vertical velocity [m/s] + Kokkos::resize(wtke_, ncol_, nlev_); + for(int i = 0; i < dropmix_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - // These are temp arrays formatted like mam4xx wants. - // Not sure if there is a way to do this with scream. - // FIXME: Do we need these? // column tendency for diagnostic output Kokkos::resize(coltend_[i], ncol_, nlev_); // column tendency @@ -1310,9 +1304,6 @@ void MAMAci::initialize_impl(const RunType run_type) { } } - for(int i = 0; i < 42; ++i) - Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); - // nact : fractional aero. number activation rate [/s] Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); @@ -1331,6 +1322,17 @@ void MAMAci::initialize_impl(const RunType run_type) { state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, mam4::aero_model::pcnst); + //--------------------------------------------------------------------------------- + // Diagnotics variables from the hetrozenous ice nucleation scheme + //--------------------------------------------------------------------------------- + + for(int i = 0; i < 42; ++i) + Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); + + //--------------------------------------------------------------------------------- + // Initialize the processes + //--------------------------------------------------------------------------------- + mam4::AeroConfig aero_config; // configure the nucleation parameterization mam4::NucleateIce::Config nucleate_ice_config; @@ -1340,6 +1342,9 @@ void MAMAci::initialize_impl(const RunType run_type) { mam4::Hetfrz::Config hetfrz_config; hetfrz_.init(aero_config, hetfrz_config); + //--------------------------------------------------------------------------------- + // Setup preprocessing + //--------------------------------------------------------------------------------- // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); From bf2e4a5c599908f3bd048205be439bf3016cf6bb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 12:07:10 -0700 Subject: [PATCH 094/142] Replaces a for with parallel for for klev --- .../mam/eamxx_mam_aci_process_interface.cpp | 66 ++++++++++--------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b18984931cd..d372a2e68f1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -648,41 +648,45 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays - // FIXME:: Kookos for here?? - for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_lev_col, klev); - - // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // create colum views of state_q - for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - } + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + KOKKOS_LAMBDA(int klev) { + // for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; + ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } - // get qqcw at a grid cell (col,lev) - // NOTE: The layout for qqcw array is based on mam_idx in dropmixnuc - // To mimic that, we are using the following for-loops - int ind_qqcw = 0; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - qqcw_view[ind_qqcw](klev) = dry_aero.cld_aero_nmr[m](icol, klev); - ++ind_qqcw; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { + // get qqcw at a grid cell (col,lev) + // NOTE: The layout for qqcw array is based on mam_idx in + // dropmixnuc To mimic that, we are using the following for-loops + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { qqcw_view[ind_qqcw](klev) = - dry_aero.cld_aero_mmr[m][a](icol, klev); + dry_aero.cld_aero_nmr[m](icol, klev); ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + qqcw_view[ind_qqcw](klev) = + dry_aero.cld_aero_mmr[m][a](icol, klev); + ++ind_qqcw; + } + } } - } - } - } + }); mam4::ndrop::dropmixnuc( team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), From 5a9c5ec50056517ac4aa1d2a0872d5788e4fd553 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 18:34:32 -0700 Subject: [PATCH 095/142] Some more cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 30 ++++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 7 ++--- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index d372a2e68f1..ba718835844 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -417,7 +417,7 @@ void compute_nucleate_ice_tendencies( const mam4::Tendencies tends(nlev); // not used const mam4::AeroConfig aero_config; const Real t = 0; // not used - nucleate_ice.compute_tendencies(aero_config, /*team,*/ t, dt, haero_atm, + nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, diags, tends); }); } @@ -792,8 +792,8 @@ void call_hetfrz_compute_tendencies( mam_coupling::AerosolState dry_aero = dry_aero_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; - MAMAci::view_2d diagnostic_scratch[42]; - for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; + MAMAci::view_2d diagnostic_scratch[43]; + for(int i = 0; i < 43; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -869,7 +869,16 @@ void call_hetfrz_compute_tendencies( diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + diags.stratiform_cloud_fraction = + ekat::subview(diagnostic_scratch[42], icol); + // assign cloud fraction + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + KOKKOS_LAMBDA(int klev) { + diags.stratiform_cloud_fraction(klev) = + haero_atm.cloud_fraction(klev); + }); //------------------------------------------------------------- // Heterogeneous freezing // frzimm, frzcnt, frzdep are the outputs of @@ -1330,7 +1339,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // Diagnotics variables from the hetrozenous ice nucleation scheme //--------------------------------------------------------------------------------- - for(int i = 0; i < 42; ++i) + for(int i = 0; i < 43; ++i) Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); //--------------------------------------------------------------------------------- @@ -1453,7 +1462,7 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); - + std::cout << " FACTNUM-2 :" << factnum_(0, 0, kb) << std::endl; //--------------------------------------------------------------- // Now update interstitial and cllud borne aerosols //--------------------------------------------------------------- @@ -1467,7 +1476,7 @@ void MAMAci::run_impl(const double dt) { update_interstitial_aerosols(team_policy, ptend_q_, nlev_, dt, // output dry_aero_); - + std::cout << " FACTNUM-1 :" << factnum_(0, 0, kb) << std::endl; // FIXME: Remove the following print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), @@ -1475,13 +1484,20 @@ void MAMAci::run_impl(const double dt) { qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), hetfrz_contact_nucleation_tend_(0, kb), hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); - + std::cout << " FACTNUM-3 :" << factnum_(0, 0, kb) << std::endl; const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); if(ans < 5.65184e-06 || ans > 5.65186e-06) { std::cout << "Somethign changed!!!! :" << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; exit(1); } + std::cout << " FACTNUM-4 :" << factnum_(0, 0, kb) << std::endl; + if((factnum_(0, 0, kb) < 0.998557 || factnum_(0, 0, kb) > 0.998559) || + (factnum_(0, 1, kb) < 0.861768 || factnum_(0, 1, kb) > 0.861770)) { + std::cout << "Somethign changed FACTNUM!!!! :" << factnum_(0, 0, kb) + << std::endl; + exit(1); + } Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index c7198818572..678ebfafbf0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -59,14 +59,13 @@ class MAMAci final : public scream::AtmosphereProcess { // aerosol dry diameter const_view_3d dgnum_; - // FIXME: Make sure these output are used somewhere and add comments about - // these view_2d nihf_; view_2d niim_; view_2d nidep_; view_2d nimey_; view_2d naai_hom_; view_2d naai_; + view_2d kvh_int_; // Eddy diffusivity of heat at the interfaces const_view_2d liqcldf_; const_view_2d liqcldf_prev_; @@ -108,7 +107,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; view_2d hetfrz_depostion_nucleation_tend_; - view_2d diagnostic_scratch_[42]; + view_2d diagnostic_scratch_[43]; // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; @@ -136,7 +135,7 @@ class MAMAci final : public scream::AtmosphereProcess { // A view array to carry cloud borne aerosol mmrs/nmrs // FIXME: 25 should be a const int - view_2d qqcw_fld_work_[25]; + view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot]; // A view to carry interstitial aerosol mmrs/nmrs view_3d state_q_work_; From 387cbf8d1664757ded7c5d81089346a1990f09c0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 18:34:49 -0700 Subject: [PATCH 096/142] Adds updated submodule --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 0f80195e0ff..00ae716b418 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 0f80195e0ff30d00d9a9b651e63a4ab4e314a242 +Subproject commit 00ae716b41881eaf47e9c437bdfdd6fc0db93952 From 918786959803cc852a12e678ab17f7ba97e317d2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 19:46:31 -0700 Subject: [PATCH 097/142] Adds the post processing step --- .../mam/eamxx_mam_aci_process_interface.cpp | 12 +++++- .../mam/eamxx_mam_aci_process_interface.hpp | 39 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ba718835844..e4a925b9835 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1358,9 +1358,13 @@ void MAMAci::initialize_impl(const RunType run_type) { //--------------------------------------------------------------------------------- // Setup preprocessing //--------------------------------------------------------------------------------- - // set up our preprocess functor + // set up our preprocess and postprocess functors preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + } // end function initialize_impl // ================================================================ @@ -1476,7 +1480,11 @@ void MAMAci::run_impl(const double dt) { update_interstitial_aerosols(team_policy, ptend_q_, nlev_, dt, // output dry_aero_); - std::cout << " FACTNUM-1 :" << factnum_(0, 0, kb) << std::endl; + + // call post processing to convert dry mixing ratios to wet mixing ratios + Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + Kokkos::fence(); + // FIXME: Remove the following print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 678ebfafbf0..d058a17c0b6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -212,9 +212,48 @@ class MAMAci final : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess + // Atmosphere processes often have a post-processing step prepares output + // from this process for the Field Manager. This functor implements this + // step, which is called during run_impl. + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_post_, nlev_post_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // MAMMicrophysics::Postprocess + private: // pre- and postprocessing scratch pads Preprocess preprocess_; + Postprocess postprocess_; }; // MAMAci From dc3e08f3a472001c1a9d03ce1d9a8fa4346a525e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 20:44:43 -0700 Subject: [PATCH 098/142] Adds top_lev as a namelist --- .../cime_config/namelist_defaults_scream.xml | 1 + .../mam/eamxx_mam_aci_process_interface.cpp | 19 ++++++---- .../mam/eamxx_mam_aci_process_interface.hpp | 37 ++++++++++++------- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 1 + .../tests/single-process/mam/aci/input.yaml | 1 + 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 2eda717ee06..49244880280 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -237,6 +237,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0.001 + 6 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e4a925b9835..983eeb7dd8d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -521,9 +521,7 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, - MAMAci::view_2d dropmixnuc_scratch_mem[15]) { - // FIXME: why can't we use MAMAci::dropmix_scratch_ above instead of 15? - + MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -792,8 +790,9 @@ void call_hetfrz_compute_tendencies( mam_coupling::AerosolState dry_aero = dry_aero_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; - MAMAci::view_2d diagnostic_scratch[43]; - for(int i = 0; i < 43; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; + MAMAci::view_2d diagnostic_scratch[MAMAci::hetro_scratch_]; + for(int i = 0; i < MAMAci::hetro_scratch_; ++i) + diagnostic_scratch[i] = diagnostic_scratch_[i]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -901,6 +900,9 @@ MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) // Asserts for the runtime or namelist options EKAT_REQUIRE_MSG(m_params.isParameter("wsubmin"), "ERROR: wsubmin is missing from mam_aci parameter list."); + EKAT_REQUIRE_MSG( + m_params.isParameter("top_level_mam4xx"), + "ERROR: top_level_mam4xx is missing from mam_aci parameter list."); } // ================================================================ @@ -1124,6 +1126,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // ------------------------------------------------------------------------ wsubmin_ = m_params.get("wsubmin"); + top_lev_ = m_params.get("top_level_mam4xx"); // ------------------------------------------------------------------------ // Input fields read in from IC file, namelist or other processes @@ -1339,7 +1342,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // Diagnotics variables from the hetrozenous ice nucleation scheme //--------------------------------------------------------------------------------- - for(int i = 0; i < 43; ++i) + for(int i = 0; i < hetro_scratch_; ++i) Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); //--------------------------------------------------------------------------------- @@ -1492,14 +1495,14 @@ void MAMAci::run_impl(const double dt) { qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), hetfrz_contact_nucleation_tend_(0, kb), hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); - std::cout << " FACTNUM-3 :" << factnum_(0, 0, kb) << std::endl; + std::cout << " FACTNUM-0 :" << factnum_(0, 0, kb) << std::endl; const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); if(ans < 5.65184e-06 || ans > 5.65186e-06) { std::cout << "Somethign changed!!!! :" << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; exit(1); } - std::cout << " FACTNUM-4 :" << factnum_(0, 0, kb) << std::endl; + std::cout << " FACTNUM-1 :" << factnum_(0, 1, kb) << std::endl; if((factnum_(0, 0, kb) < 0.998557 || factnum_(0, 0, kb) > 0.998559) || (factnum_(0, 1, kb) < 0.861768 || factnum_(0, 1, kb) > 0.861770)) { std::cout << "Somethign changed FACTNUM!!!! :" << factnum_(0, 0, kb) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index d058a17c0b6..0bfbefc118f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -19,6 +19,12 @@ namespace scream { class MAMAci final : public scream::AtmosphereProcess { + public: + // declare some constant scratch space lengths + static constexpr int hetro_scratch_ = 43; + static constexpr int dropmix_scratch_ = 15; + + private: using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; @@ -32,11 +38,16 @@ class MAMAci final : public scream::AtmosphereProcess { using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; - // FIXME: Should the following variables be public? They are like that in - // micriphysics and optics codes - + //------------------------------------------------------------------------ // ACI runtime ( or namelist) options - Real wsubmin_; + //------------------------------------------------------------------------ + + Real wsubmin_; // Minimum subgrid vertical velocity + int top_lev_; // Top level for MAM4xx + + //------------------------------------------------------------------------ + // END: ACI runtime ( or namelist) options + //------------------------------------------------------------------------ // rho is air density [kg/m3] view_2d rho_; @@ -59,13 +70,17 @@ class MAMAci final : public scream::AtmosphereProcess { // aerosol dry diameter const_view_3d dgnum_; + // ice nucleation diagnostic variables view_2d nihf_; view_2d niim_; view_2d nidep_; view_2d nimey_; view_2d naai_hom_; + + // ice nucleation output for FM view_2d naai_; + // droplet activation inputs and outputs view_2d kvh_int_; // Eddy diffusivity of heat at the interfaces const_view_2d liqcldf_; const_view_2d liqcldf_prev_; @@ -74,7 +89,6 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d cloud_frac_; view_2d cloud_frac_prev_; view_2d qcld_; - view_2d tendnd_; view_2d ptend_q_[mam4::aero_model::pcnst]; view_3d factnum_; const_view_3d qqcw_input_; @@ -94,26 +108,21 @@ class MAMAci final : public scream::AtmosphereProcess { view_3d nact_; view_3d mact_; - - static constexpr int dropmix_scratch_ = 15; view_2d dropmixnuc_scratch_mem_[dropmix_scratch_]; - view_2d stratiform_cloud_fraction_; - view_2d activation_fraction_accum_idx_; - view_2d activation_fraction_coarse_idx_; + // droplet activation output for the FM + view_2d tendnd_; // These are the output tendencies from heterogeneous freezing that need to be // added correctly to the cloud-micorphysics scheme. view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; view_2d hetfrz_depostion_nucleation_tend_; - view_2d diagnostic_scratch_[43]; + + view_2d diagnostic_scratch_[hetro_scratch_]; // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; - // Top level for troposphere cloud physics - // FIXME: This should be read in to make user selectable. - const int top_lev_ = 6; // local atmospheric state column variables const_view_2d pdel_; // pressure thickess of layer [Pa] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index 5cc2fb8fa65..a8cac4b1a6c 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -34,6 +34,7 @@ atmosphere_processes: Ckm: 0.1 mam4_aci: wsubmin: 0.001 + top_level_mam4xx: 6 grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index bedc9c583c2..3e778315bdf 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -12,6 +12,7 @@ atmosphere_processes: atm_procs_list: [mam4_aci] mam4_aci: wsubmin: 0.001 + top_level_mam4xx: 6 grids_manager: Type: Mesh Free From d6f8a940e768db8315922010ef82e44d115d92f3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 21:10:40 -0700 Subject: [PATCH 099/142] Repaces parallel_for copy logic with deep_copy --- .../mam/eamxx_mam_aci_process_interface.cpp | 49 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 - 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 983eeb7dd8d..4689e88ca07 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -71,6 +71,16 @@ void print_output(const Real w0, const Real rho, const Real tke, } } + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + std::cout << "hetfrz_immersion_nucleation_tend_:" << hetfrz_immersion_nucleation_tend << ":" << hetfrz_immersion_nucleation_tend << std::endl; @@ -725,26 +735,17 @@ void update_cloud_borne_aerosols( const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, // output mam_coupling::AerosolState &dry_aero) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { - int ind_qqcw = 0; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - dry_aero.cld_aero_nmr[m](icol, kk) = - qqcw_fld_work[ind_qqcw](icol, kk); - ++ind_qqcw; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](icol, kk) = - qqcw_fld_work[ind_qqcw](icol, kk); - ++ind_qqcw; - } - } - } - }); - }); + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], qqcw_fld_work[ind_qqcw]); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], qqcw_fld_work[ind_qqcw]); + ++ind_qqcw; + } + } + } } // Update interstitial aerosols using tendencies @@ -1381,17 +1382,13 @@ void MAMAci::run_impl(const double dt) { // kb)); const auto scan_policy = ekat::ExeSpaceUtils< - // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, - // nlev_); - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); - + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 0bfbefc118f..7538451db6e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -143,7 +143,6 @@ class MAMAci final : public scream::AtmosphereProcess { std::shared_ptr grid_; // A view array to carry cloud borne aerosol mmrs/nmrs - // FIXME: 25 should be a const int view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot]; // A view to carry interstitial aerosol mmrs/nmrs From c76b4260fa706a6adbf748a7d258b16a058d03a7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 22:29:48 -0700 Subject: [PATCH 100/142] Add function call to get interface values for wsec and kvh --- .../mam/eamxx_mam_aci_process_interface.cpp | 240 ++++-------------- .../tests/single-process/mam/aci/input.yaml | 7 +- 2 files changed, 51 insertions(+), 196 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4689e88ca07..b98421ca233 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -4,23 +4,6 @@ namespace scream { namespace { -void print_input(const Real t, const Real p, const Real wsec) { - std::cout << "T_mid:" << t << std::endl; - std::cout << "p_mid:" << p << std::endl; - std::cout << "wsec:" << wsec << std::endl; -} -void print_bef_ndrop(const mam_coupling::AerosolState &dry_aero, const int kb) { - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "Bef-ndrop-cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) - << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "Bef-ndrop-cldbrn-mmr:" - << dry_aero.cld_aero_mmr[m][a](0, kb) << std::endl; - } - } - } -} void print_output(const Real w0, const Real rho, const Real tke, const Real wsub, const Real wice, const Real wsig, const Real naai_hom, const Real naai, const Real rpdel, @@ -90,148 +73,6 @@ void print_output(const Real w0, const Real rho, const Real tke, << hetfrz_depostion_nucleation_tend << std::endl; } -void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, - const int ncol_, const int nlev_) { - const Real w_sec_e3sm[73] = { - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.44201892319518146E-003, - 0.77315620137962326E-003, 0.24806301482800117E-002, - 0.11109772692898754E-001, 0.40651094032490273E-001, - 0.82156694426095800E-001, 0.12207124453993526E+000, - 0.15516728994634110E+000, 0.17775318086169636E+000, - 0.18549817250146838E+000, 0.17184548286554119E+000, - 0.12741230682196053E+000, 0.65495229516041628E-001, - 0.26909155217660592E-001}; - - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_e3sm[kk]; - // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); - } - // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); - w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; - } - const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, - 0.75991761081225006E-002, 0.12291092068185365E-001, - 0.11484807652762415E-001, 0.10856880396302943E-001, - 0.10500384508819637E-001, 0.10361486171738229E-001, - 0.10333325067964508E-001, 0.10258838031435397E-001, - 0.10027325248446619E-001, 0.97784259072973521E-002, - 0.96611845055866539E-002, 0.96249746122327937E-002, - 0.95773431515696512E-002, 0.95180614513688099E-002, - 0.94713233348487150E-002, 0.94503864489758338E-002, - 0.94536294366578833E-002, 0.94575972194308883E-002, - 0.94403767489615684E-002, 0.93975694769176284E-002, - 0.93322843554751022E-002, 0.92777070192527501E-002, - 0.92456776697171228E-002, 0.92266924824142716E-002, - 0.92123025773060436E-002, 0.91888715633294191E-002, - 0.91516797753615851E-002, 0.90958299606649744E-002, - 0.89988037524983237E-002, 0.88220984587642423E-002, - 0.85231270833157156E-002, 0.81397522619395188E-002, - 0.79160421807845088E-002, 0.81206851117902653E-002, - 0.86526891616674779E-002, 0.91682975412125615E-002, - 0.96043394254592580E-002, 0.10033689085881327E-001, - 0.10428656694074272E-001, 0.10715913043864789E-001, - 0.10919631245454951E-001, 0.11250937075285789E-001, - 0.11829292157343831E-001, 0.12413311776454055E-001, - 0.12851317662157077E-001, 0.13175523677700330E-001, - 0.13224182907540188E-001, 0.13085937680733115E-001, - 0.12615055546741534E-001, 0.11995423733019836E-001, - 0.12346556881757400E-001, 0.13433752971524651E-001, - 0.13904308240950175E-001, 0.13539811748121957E-001, - 0.12555099320041433E-001, 0.11519643673351362E-001, - 0.11414071302852231E-001, 0.13409756835238139E-001, - 0.24071962815959351E-001, 0.75489419450816414E-001, - 0.62082011878960308E+000, 0.63952862312816796E+001, - 0.16226857944175123E+002, 0.21882852534279891E+002, - 0.24966173574402408E+002, 0.25710753126453692E+002, - 0.24069881024271943E+002, 0.19743922403487922E+002, - 0.98667814246712027E+001, 0.25633359450143991E+001, - 0.14682471685037493E+001}; - // compute eddy diffusivity of heat at the interfaces - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - // kvh_int_(icol, kk) = kvh_mid_(icol, kk); - kvh_int_(icol, kk) = kvh_e3sm[kk]; - } - // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); - kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; - } -} -void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, - const int nlev_) { - const Real dgnum_ait_e3sm[72] = { - 0.20877713336487552E-007, 0.21782230353342090E-007, - 0.21688324003865861E-007, 0.21112855042342451E-007, - 0.19162058462939536E-007, 0.18102979880838476E-007, - 0.17906980715477606E-007, 0.20271254074583327E-007, - 0.22698983422181942E-007, 0.24134835117044986E-007, - 0.25498156808001372E-007, 0.29796738799905547E-007, - 0.35822987394021908E-007, 0.41170963764365215E-007, - 0.44892726528330642E-007, 0.47217231342203107E-007, - 0.48928661807108766E-007, 0.50170939816128735E-007, - 0.51078750853732200E-007, 0.52247333465736065E-007, - 0.53190758580174931E-007, 0.53576491941850044E-007, - 0.53915614473890715E-007, 0.54510964775236826E-007, - 0.55643231691556703E-007, 0.57057811112589899E-007, - 0.58177383586181116E-007, 0.58209849180850108E-007, - 0.57976751598840998E-007, 0.52000000000000002E-007, - 0.50728746567226150E-007, 0.49119902704480870E-007, - 0.48212162162050883E-007, 0.49227715213506454E-007, - 0.46876827233752246E-007, 0.45360603896257791E-007, - 0.49986783979004747E-007, 0.51186879246229022E-007, - 0.50009353247048599E-007, 0.48250264542204811E-007, - 0.47560278748093609E-007, 0.48298089720730957E-007, - 0.49095935613468768E-007, 0.49493024126912931E-007, - 0.50250797590476007E-007, 0.51949267668322422E-007, - 0.53778727208416418E-007, 0.53563593301099588E-007, - 0.51218136771199298E-007, 0.43171429694325200E-007, - 0.39019610039033895E-007, 0.36175109143257051E-007, - 0.42731638777892750E-007, 0.38060728507221777E-007, - 0.44046323901481340E-007, 0.39216732751330010E-007, - 0.34842233953609988E-007, 0.34068804733226066E-007, - 0.30636043694263528E-007, 0.28302341686131413E-007, - 0.33023014309036320E-007, 0.34745748365385196E-007, - 0.43623545003583371E-007, 0.48206451795644064E-007, - 0.49854490325455530E-007, 0.50346335647724146E-007, - 0.50661560988561763E-007, 0.50986261962838767E-007, - 0.51256955985111086E-007, 0.51482578449096488E-007, - 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - } - } -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -273,6 +114,29 @@ void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, }); } +void compute_values_at_interfaces(haero::ThreadTeamPolicy team_policy, + const MAMAci::const_view_2d var_mid, + const MAMAci::view_2d dz, const int nlev_, + // output + MAMAci::view_2d var_int) { + using CO = scream::ColumnOps; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + const auto var_mid_col = ekat::subview(var_mid, icol); + const auto var_int_col = ekat::subview(var_int, icol); + const auto dz_col = ekat::subview(dz, icol); + + const Real bc_top = var_mid_col(0); + const Real bc_bot = var_mid_col(nlev_ - 1); + + CO::compute_interface_values_linear(team, nlev_, var_mid_col, dz_col, + bc_top, bc_bot, var_int_col); + }); +} + KOKKOS_INLINE_FUNCTION void compute_tke_using_w_sec(const haero::ThreadTeam &team, const MAMAci::const_view_2d w_sec, const int icol, @@ -989,11 +853,8 @@ void MAMAci::set_grids( // the current time step. For other dycores (such as EUL), it may be different // and we might need to revisit this. - // FIXME: w_variance in microp_aero_run.F90 is at the interfaces but - // SHOC provides it at the midpoints. Verify how it is being used. - // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -1007,8 +868,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - // FIXME: It is at mid level in EAMxx, we need to compute it at the interfaces - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -1360,7 +1220,7 @@ void MAMAci::initialize_impl(const RunType run_type) { hetfrz_.init(aero_config, hetfrz_config); //--------------------------------------------------------------------------------- - // Setup preprocessing + // Setup preprocessing and post processing //--------------------------------------------------------------------------------- // set up our preprocess and postprocess functors preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, @@ -1375,14 +1235,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - // FIXME: Remove set_input and print_input - const int kb = 62; - set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - // print_input(dry_atm_.T_mid(0, kb), dry_atm_.p_mid(0, kb), w_sec_int_(0, - // kb)); - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -1390,10 +1245,19 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + const int itop = + (dry_atm_.p_mid(0, 0) < dry_atm_.p_mid(0, nlev_ - 1)) ? 0 : nlev_ - 1; + std::cout << "ITOP::::::::" << itop << std::endl; + compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); + // Get w_sec_int_ from w_sec_mid_ + compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + // output + w_sec_int_); + compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output tke_); @@ -1409,8 +1273,6 @@ void MAMAci::run_impl(const double dt) { compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - // FIXME:Remove set_dgait - set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -1436,7 +1298,12 @@ void MAMAci::run_impl(const double dt) { rpdel_); Kokkos::fence(); // wait for rpdel_ to be computed. - // print_bef_ndrop(dry_aero_, kb); + + // Get kvh_int_ from kvh_mid_ + compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + // output + kvh_int_); + // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial // aerosols tendencies @@ -1466,9 +1333,9 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); - std::cout << " FACTNUM-2 :" << factnum_(0, 0, kb) << std::endl; + //--------------------------------------------------------------- - // Now update interstitial and cllud borne aerosols + // Now update interstitial and cloud borne aerosols //--------------------------------------------------------------- // Update cloud borne aerosols @@ -1483,30 +1350,15 @@ void MAMAci::run_impl(const double dt) { // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); - Kokkos::fence(); + Kokkos::fence(); // wait before returning to calling function - // FIXME: Remove the following + const int kb = 62; print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), hetfrz_contact_nucleation_tend_(0, kb), hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); - std::cout << " FACTNUM-0 :" << factnum_(0, 0, kb) << std::endl; - const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); - if(ans < 5.65184e-06 || ans > 5.65186e-06) { - std::cout << "Somethign changed!!!! :" - << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; - exit(1); - } - std::cout << " FACTNUM-1 :" << factnum_(0, 1, kb) << std::endl; - if((factnum_(0, 0, kb) < 0.998557 || factnum_(0, 0, kb) > 0.998559) || - (factnum_(0, 1, kb) < 0.861768 || factnum_(0, 1, kb) > 0.861770)) { - std::cout << "Somethign changed FACTNUM!!!! :" << factnum_(0, 0, kb) - << std::endl; - exit(1); - } - Kokkos::fence(); // wait before returning to calling function } } // namespace scream \ No newline at end of file diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index 3e778315bdf..2c4172b0da1 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -26,14 +26,17 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file #we should get the following variables from other processes pbl_height : 1.0 - dgnum: 1e-3 + dgnum: 4.362354500358337E-008 + cldfrac_liq_prev: 0.138584624960092 + eddy_diff_heat: 0.620820118789603 + w_variance: 1.110977269289875E-002 # The parameters for I/O control Scorpio: From 21cba9ddfa8ba187ebd0369464324f046c0a3606 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 13:08:19 -0700 Subject: [PATCH 101/142] Removes unused field tags and add missing namelist vars in the intput.yaml files --- .../mam/eamxx_mam_aci_process_interface.cpp | 24 +++++++--- .../eamxx/src/share/field/field_tag.hpp | 45 +++++++------------ .../input.yaml | 3 ++ .../input.yaml | 3 ++ .../physics_only/mam/shoc_mam4_aci/input.yaml | 3 ++ 5 files changed, 41 insertions(+), 37 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b98421ca233..d285a42748c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,5 +1,19 @@ #include +/* NOTES: +1. w_variance assumes that we are using SE dycore. If we use EUL +dycore, we need to get its value from previous dynamic time step + +2. w_variance and eddy_diff_heat are at midpoints in EAMxx, we +derived their interface values using boundary conditions to be +the top and botton values of the mid point arrays. We assume that +this assumption should not cause any issues. + +FUTURE WORK: +1. MAM4xx submodule should point to MAM4xx main branch +2. Link hetrozenous freezing outputs to microphysics +*/ + namespace scream { namespace { @@ -949,15 +963,15 @@ void MAMAci::set_grids( // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); - // heterogeous freezing by immersion nucleation [cm^-3 s^-1] + // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); - // heterogeous freezing by contact nucleation [cm^-3 s^-1] + // heterogeneous freezing by contact nucleation [cm^-3 s^-1] add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); - // heterogeous freezing by deposition nucleation [cm^-3 s^-1] + // heterogeneous freezing by deposition nucleation [cm^-3 s^-1] add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); } // function set_grids ends @@ -1245,10 +1259,6 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - const int itop = - (dry_atm_.p_mid(0, 0) < dry_atm_.p_mid(0, nlev_ - 1)) ? 0 : nlev_ - 1; - std::cout << "ITOP::::::::" << itop << std::endl; - compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index 7e6b0b83412..a2e19662162 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -34,15 +34,12 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, - num_modes, - num_constituents, - num_aero_const, - psat, // - nrefindex_real, - nrefindex_im, - ncoef_number, - mode + MAM_NumModes, + MAM_NumRefIndexReal, + MAM_NumRefIndexImag, + MAM_NumCoefficients, + MAM_NumModesInFile }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -59,16 +56,13 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; - constexpr auto NMODES = FieldTag::num_modes; - constexpr auto MAM_NCNST = FieldTag::num_constituents; - constexpr auto MAM_AERO_NCNST = FieldTag::num_aero_const; - constexpr auto MAM_PSAT = FieldTag::psat; + constexpr auto NMODES = FieldTag::MAM_NumModes; // - constexpr auto NREFINDEX_REAL = FieldTag::nrefindex_real; - constexpr auto NREFINDEX_IM = FieldTag::nrefindex_im; + constexpr auto NREFINDEX_REAL = FieldTag::MAM_NumRefIndexReal; + constexpr auto NREFINDEX_IM = FieldTag::MAM_NumRefIndexImag; - constexpr auto NCOEF_NUMBER = FieldTag::ncoef_number; - constexpr auto MODE = FieldTag::mode; + constexpr auto NCOEF_NUMBER = FieldTag::MAM_NumCoefficients; + constexpr auto MODE = FieldTag::MAM_NumModesInFile; } inline std::string e2str (const FieldTag ft) { @@ -99,28 +93,19 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; - case FieldTag::num_modes: + case FieldTag::MAM_NumModes: name = "num_modes"; break; - case FieldTag::num_constituents: - name = "num_constituents"; - break; - case FieldTag::num_aero_const: - name = "num_aero_const"; - break; - case FieldTag::psat: - name = "psat"; - break; - case FieldTag::nrefindex_real: + case FieldTag::MAM_NumRefIndexReal: name = "refindex_real"; break; - case FieldTag::nrefindex_im: + case FieldTag::MAM_NumRefIndexImag: name = "refindex_im"; break; - case FieldTag::ncoef_number: + case FieldTag::MAM_NumCoefficients: name = "coef_number"; break; - case FieldTag::mode: + case FieldTag::MAM_NumModesInFile: name = "mode"; break; default: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml index f3198b1308a..20d6f142c52 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml @@ -16,6 +16,9 @@ atmosphere_processes: Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 p3: max_total_ni: 740.0e3 do_prescribed_ccn: false diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml index d61f1318f8f..7ec317eda92 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml @@ -16,6 +16,9 @@ atmosphere_processes: Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 p3: max_total_ni: 740.0e3 do_prescribed_ccn: false diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml index 1d6d4f48eba..ae997bb3da5 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -16,6 +16,9 @@ atmosphere_processes: Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 shoc: lambda_low: 0.001 lambda_high: 0.04 From a4be737029decba2452592c6c31bfcb50c6536b7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 13:23:55 -0700 Subject: [PATCH 102/142] Adds a CIME test --- .../testmods_dirs/scream/mam4xx/aci/shell_commands | 9 +++++++++ .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands new file mode 100644 index 00000000000..cde7694f5f3 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -0,0 +1,9 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41 +./xmlchange SCREAM_CMAKE_OPTIONS="`./xmlquery --value SCREAM_CMAKE_OPTIONS | sed 's/SCREAM_NUM_TRACERS [0-9][0-9]*/SCREAM_NUM_TRACERS 41/'`" + +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index d285a42748c..d2ddf4436ad 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -4,7 +4,7 @@ 1. w_variance assumes that we are using SE dycore. If we use EUL dycore, we need to get its value from previous dynamic time step -2. w_variance and eddy_diff_heat are at midpoints in EAMxx, we +2. w_variance and eddy_diff_heat are at midpoints in EAMxx, we derived their interface values using boundary conditions to be the top and botton values of the mid point arrays. We assume that this assumption should not cause any issues. From cd1f99a3fc6cbb21024c9bb4eaee241c33c6158a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 14:10:04 -0700 Subject: [PATCH 103/142] Removes print statements and recomputes dry/wet mmrs in mam_coupling --- .../mam/eamxx_mam_aci_process_interface.cpp | 82 +------------------ .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 20 ++--- 3 files changed, 15 insertions(+), 89 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index d2ddf4436ad..f1e7b320cb6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,6 +1,8 @@ #include -/* NOTES: +/* +----------------------------------------------------------------- +NOTES: 1. w_variance assumes that we are using SE dycore. If we use EUL dycore, we need to get its value from previous dynamic time step @@ -12,81 +14,13 @@ this assumption should not cause any issues. FUTURE WORK: 1. MAM4xx submodule should point to MAM4xx main branch 2. Link hetrozenous freezing outputs to microphysics +----------------------------------------------------------------- */ namespace scream { namespace { -void print_output(const Real w0, const Real rho, const Real tke, - const Real wsub, const Real wice, const Real wsig, - const Real naai_hom, const Real naai, const Real rpdel, - MAMAci::view_3d factnum, const Real tendnd, - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - const Real hetfrz_immersion_nucleation_tend, - const Real hetfrz_contact_nucleation_tend, - const Real hetfrz_depostion_nucleation_tend, - const mam_coupling::AerosolState &dry_aero, const int kb) { - std::cout << "w0:" << w0 << std::endl; - std::cout << " rho: " << rho << std::endl; - std::cout << "TKE:" << tke << std::endl; - std::cout << "WSUB:" << wsub << std::endl; - std::cout << "WICE:" << wice << std::endl; - std::cout << "WSIG:" << wsig << std::endl; - std::cout << "naai_hom_:" << naai_hom << std::endl; - std::cout << "naai_:" << naai << std::endl; - std::cout << "rpdel_:" << rpdel << std::endl; - std::cout << "factnum_:" << factnum(0, 0, kb) << " : " << factnum(0, 1, kb) - << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) - << std::endl; - std::cout << "tendnd_:" << tendnd << std::endl; - for(int ic = 9; ic < 40; ++ic) { - std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; - } - for(int ic = 0; ic < 25; ++ic) { - std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) - << std::endl; - } - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend << ":" - << hetfrz_immersion_nucleation_tend << std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" - << hetfrz_contact_nucleation_tend << std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" - << hetfrz_depostion_nucleation_tend << std::endl; -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -1361,14 +1295,6 @@ void MAMAci::run_impl(const double dt) { // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function - - const int kb = 62; - print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), - wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), - rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, - qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), - hetfrz_contact_nucleation_tend_(0, kb), - hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); } } // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 7538451db6e..974907ee8c0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -256,7 +256,7 @@ class MAMAci final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_post_; mam_coupling::DryAtmosphere dry_atm_post_; mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; - }; // MAMMicrophysics::Postprocess + }; // Postprocess private: // pre- and postprocessing scratch pads diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 25cc9d5de9d..6b5cfbd186d 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -644,11 +644,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -669,21 +669,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);// PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } From 94a80d9664db5a69538be91011ecebf2a5bc175d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 14:56:30 -0700 Subject: [PATCH 104/142] Revives small kernels shoc function, changes shell cmd for CIME test, fix test runtimes --- .../testmods_dirs/scream/mam4xx/aci/shell_commands | 4 ++-- .../physics/shoc/disp/shoc_assign_2d_view_disp.cpp | 4 ++-- .../physics/shoc/eamxx_shoc_process_interface.cpp | 12 ++++++------ .../eamxx/src/physics/shoc/impl/shoc_main_impl.hpp | 2 +- .../mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt | 3 +-- .../tests/single-process/mam/aci/CMakeLists.txt | 2 +- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index cde7694f5f3..3bffab45ba6 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -1,7 +1,7 @@ #Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41 -./xmlchange SCREAM_CMAKE_OPTIONS="`./xmlquery --value SCREAM_CMAKE_OPTIONS | sed 's/SCREAM_NUM_TRACERS [0-9][0-9]*/SCREAM_NUM_TRACERS 41/'`" +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b diff --git a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp index 8f5592a3836..e6cd04054d0 100644 --- a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp +++ b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp @@ -13,7 +13,7 @@ ::shoc_assign_2d_view_disp( const view_2d& input_view, const view_2d& output_view) { - /*using ExeSpace = typename KT::ExeSpace; + using ExeSpace = typename KT::ExeSpace; const auto nlev_packs = ekat::npack(nlev); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nlev_packs); @@ -22,7 +22,7 @@ ::shoc_assign_2d_view_disp( Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { output_view (i,k) = input_view(i,k); }); - });*/ + }); } } // namespace shoc diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 6059a4369ed..25c2674c912 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -80,12 +80,12 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables add_field("pbl_height", scalar2d_layout_col, m, grid_name); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 34a0455c14f..ca163a49ddd 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -266,7 +266,7 @@ void Functions::shoc_main_internal( check_tke(team,nlev,tke); } - // End SHOC parameterization + // End SHOC parameterization // Use SHOC outputs to update the host model // temperature diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt index 81eaa3ae3e4..946d68cb750 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt @@ -13,8 +13,7 @@ CreateADUnitTest(${TEST_BASE_NAME} # Set AD configurable options set (ATM_TIME_STEP 1800) -#SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) # 1h 4h 24h +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h set (RUN_T0 2021-10-12-45000) # Determine num subcycles needed to keep shoc dt<=300s diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 95da48b7fa4..b175e6f5e7c 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,7 +12,7 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) set (ATM_TIME_STEP 600) set (RUN_T0 2021-10-12-45000) From 888fdb2fbc2b0cfb05e743c8b4a3976c6387ff93 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 16:43:08 -0700 Subject: [PATCH 105/142] Adds int def after reabse --- components/eamxx/src/physics/mam/mam_coupling.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 6b5cfbd186d..c2fcfb6a8ea 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -35,6 +35,8 @@ using uview_2d = typename ekat::template Unmanaged; +using view_int_1d = typename KT::template view_1d; + // number of constituents in gas chemistry "work arrays" KOKKOS_INLINE_FUNCTION constexpr int gas_pcnst() { From 7e33cdac4862980b4b7afbed4771cbe9d7d6c723 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Apr 2024 14:58:39 -0700 Subject: [PATCH 106/142] Fixes GNU compilation: Make views public and removed an unused variable --- .../mam/eamxx_mam_aci_process_interface.cpp | 3 +-- .../mam/eamxx_mam_aci_process_interface.hpp | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index f1e7b320cb6..016e0c5b5fe 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -473,7 +473,6 @@ void call_function_dropmixnuc( KOKKOS_LAMBDA(int klev) { // for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) // NOTE: The order of species in state_q_at_lev_col @@ -1297,4 +1296,4 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait before returning to calling function } -} // namespace scream \ No newline at end of file +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 974907ee8c0..394ab89f21e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -24,19 +24,21 @@ class MAMAci final : public scream::AtmosphereProcess { static constexpr int hetro_scratch_ = 43; static constexpr int dropmix_scratch_ = 15; + // views for multi-column data + using view_2d = scream::mam_coupling::view_2d; + using const_view_2d = scream::mam_coupling::const_view_2d; + using view_3d = scream::mam_coupling::view_3d; + using const_view_3d = scream::mam_coupling::const_view_3d; + private: using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; mam4::Hetfrz hetfrz_; - // views for single- and multi-column data + // views for single-column data using view_1d = scream::mam_coupling::view_1d; - using view_2d = scream::mam_coupling::view_2d; - using view_3d = scream::mam_coupling::view_3d; using const_view_1d = scream::mam_coupling::const_view_1d; - using const_view_2d = scream::mam_coupling::const_view_2d; - using const_view_3d = scream::mam_coupling::const_view_3d; //------------------------------------------------------------------------ // ACI runtime ( or namelist) options @@ -153,19 +155,19 @@ class MAMAci final : public scream::AtmosphereProcess { MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // Process metadata: Return type of the process - AtmosphereProcessType MAMAci::type() const { + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } // Return name of the process - std::string MAMAci::name() const { return "mam4_aci"; } + std::string name() const { return "mam4_aci"; } // grid void set_grids( const std::shared_ptr grids_manager) override; // management of common atm process memory - size_t MAMAci::requested_buffer_size_in_bytes() const { + size_t requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } From 9f54b0d5f593dd58c9995ff0d2c73a5153939adb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Apr 2024 17:00:32 -0700 Subject: [PATCH 107/142] PM-GPU: Fixes nested Kokkos loops compilation --- .../physics/mam/eamxx_mam_aci_process_interface.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 016e0c5b5fe..3b4ce152135 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -469,9 +469,9 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - KOKKOS_LAMBDA(int klev) { - // for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { + Kokkos::TeamThreadRange(team, 0u, 72),//mam4::ndrop::pver), + [=](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -570,7 +570,7 @@ void update_interstitial_aerosols( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamThreadRange(team, nlev), [=](int kk) { int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { for(int a = 0; a < mam4::num_species_mode(m); ++a) { @@ -686,7 +686,7 @@ void call_hetfrz_compute_tendencies( // assign cloud fraction Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - KOKKOS_LAMBDA(int klev) { + [=](int klev) { diags.stratiform_cloud_fraction(klev) = haero_atm.cloud_fraction(klev); }); From 365325c70ac14ec77dd2a1dae2302ad9dcba0ce9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Apr 2024 17:17:19 -0700 Subject: [PATCH 108/142] Removes a debug change --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 3b4ce152135..9c0c99cea06 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -469,7 +469,7 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, 72),//mam4::ndrop::pver), + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), [=](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; From 727cc8ac80fc36d5256cc8c7d7cc07e30dc8ce01 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 30 Apr 2024 11:55:01 -0700 Subject: [PATCH 109/142] Changes needed for single cell validation --- .../mam/eamxx_mam_aci_process_interface.cpp | 416 ++++++++++++++++-- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 20 +- .../single-process/mam/aci/CMakeLists.txt | 3 +- .../tests/single-process/mam/aci/input.yaml | 11 +- 5 files changed, 403 insertions(+), 51 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 9c0c99cea06..9e3c272ef89 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -21,6 +21,217 @@ namespace scream { namespace { +void print_output(const Real w0, const Real rho, const Real tke, + const Real wsub, const Real wice, const Real wsig, + const Real naai_hom, const Real naai, const Real rpdel, + MAMAci::view_3d factnum, const Real tendnd, + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + const Real hetfrz_immersion_nucleation_tend, + const Real hetfrz_contact_nucleation_tend, + const Real hetfrz_depostion_nucleation_tend, + const mam_coupling::AerosolState &dry_aero, const int kb) { + std::cout << "w0:" << w0 << std::endl; + std::cout << " rho: " << rho << std::endl; + std::cout << "TKE:" << tke << std::endl; + std::cout << "WSUB:" << wsub << std::endl; + std::cout << "WICE:" << wice << std::endl; + std::cout << "WSIG:" << wsig << std::endl; + std::cout << "naai_hom_:" << naai_hom << std::endl; + std::cout << "naai_:" << naai << std::endl; + std::cout << "rpdel_:" << rpdel << std::endl; + std::cout << "factnum_:" << factnum(0, 0, kb) << " : " << factnum(0, 1, kb) + << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) + << std::endl; + std::cout << "tendnd_:" << tendnd << std::endl; + for(int ic = 9; ic < 40; ++ic) { + std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; + } + for(int ic = 0; ic < 25; ++ic) { + std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) + << std::endl; + } + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + std::cout << "hetfrz_immersion_nucleation_tend_:" + << hetfrz_immersion_nucleation_tend << ":" + << hetfrz_immersion_nucleation_tend << std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" + << hetfrz_contact_nucleation_tend << std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" + << hetfrz_depostion_nucleation_tend << std::endl; +} + +void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, + const int ncol_, const int nlev_) { + const Real w_sec_e3sm[73] = { + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.44201892319518146E-003, + 0.77315620137962326E-003, 0.24806301482800117E-002, + 0.11109772692898754E-001, 0.40651094032490273E-001, + 0.82156694426095800E-001, 0.12207124453993526E+000, + 0.15516728994634110E+000, 0.17775318086169636E+000, + 0.18549817250146838E+000, 0.17184548286554119E+000, + 0.12741230682196053E+000, 0.65495229516041628E-001, + 0.26909155217660592E-001}; + + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + w_sec_int_(icol, kk) = w_sec_e3sm[kk]; + // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + } + // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; + } + const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, + 0.75991761081225006E-002, 0.12291092068185365E-001, + 0.11484807652762415E-001, 0.10856880396302943E-001, + 0.10500384508819637E-001, 0.10361486171738229E-001, + 0.10333325067964508E-001, 0.10258838031435397E-001, + 0.10027325248446619E-001, 0.97784259072973521E-002, + 0.96611845055866539E-002, 0.96249746122327937E-002, + 0.95773431515696512E-002, 0.95180614513688099E-002, + 0.94713233348487150E-002, 0.94503864489758338E-002, + 0.94536294366578833E-002, 0.94575972194308883E-002, + 0.94403767489615684E-002, 0.93975694769176284E-002, + 0.93322843554751022E-002, 0.92777070192527501E-002, + 0.92456776697171228E-002, 0.92266924824142716E-002, + 0.92123025773060436E-002, 0.91888715633294191E-002, + 0.91516797753615851E-002, 0.90958299606649744E-002, + 0.89988037524983237E-002, 0.88220984587642423E-002, + 0.85231270833157156E-002, 0.81397522619395188E-002, + 0.79160421807845088E-002, 0.81206851117902653E-002, + 0.86526891616674779E-002, 0.91682975412125615E-002, + 0.96043394254592580E-002, 0.10033689085881327E-001, + 0.10428656694074272E-001, 0.10715913043864789E-001, + 0.10919631245454951E-001, 0.11250937075285789E-001, + 0.11829292157343831E-001, 0.12413311776454055E-001, + 0.12851317662157077E-001, 0.13175523677700330E-001, + 0.13224182907540188E-001, 0.13085937680733115E-001, + 0.12615055546741534E-001, 0.11995423733019836E-001, + 0.12346556881757400E-001, 0.13433752971524651E-001, + 0.13904308240950175E-001, 0.13539811748121957E-001, + 0.12555099320041433E-001, 0.11519643673351362E-001, + 0.11414071302852231E-001, 0.13409756835238139E-001, + 0.24071962815959351E-001, 0.75489419450816414E-001, + 0.62082011878960308E+000, 0.63952862312816796E+001, + 0.16226857944175123E+002, 0.21882852534279891E+002, + 0.24966173574402408E+002, 0.25710753126453692E+002, + 0.24069881024271943E+002, 0.19743922403487922E+002, + 0.98667814246712027E+001, 0.25633359450143991E+001, + 0.14682471685037493E+001}; + // compute eddy diffusivity of heat at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + // kvh_int_(icol, kk) = kvh_mid_(icol, kk); + kvh_int_(icol, kk) = kvh_e3sm[kk]; + } + // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; + } +} +void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, + const int nlev_) { + const Real dgnum_ait_e3sm[72] = { + 0.20877713336487552E-007, 0.21782230353342090E-007, + 0.21688324003865861E-007, 0.21112855042342451E-007, + 0.19162058462939536E-007, 0.18102979880838476E-007, + 0.17906980715477606E-007, 0.20271254074583327E-007, + 0.22698983422181942E-007, 0.24134835117044986E-007, + 0.25498156808001372E-007, 0.29796738799905547E-007, + 0.35822987394021908E-007, 0.41170963764365215E-007, + 0.44892726528330642E-007, 0.47217231342203107E-007, + 0.48928661807108766E-007, 0.50170939816128735E-007, + 0.51078750853732200E-007, 0.52247333465736065E-007, + 0.53190758580174931E-007, 0.53576491941850044E-007, + 0.53915614473890715E-007, 0.54510964775236826E-007, + 0.55643231691556703E-007, 0.57057811112589899E-007, + 0.58177383586181116E-007, 0.58209849180850108E-007, + 0.57976751598840998E-007, 0.52000000000000002E-007, + 0.50728746567226150E-007, 0.49119902704480870E-007, + 0.48212162162050883E-007, 0.49227715213506454E-007, + 0.46876827233752246E-007, 0.45360603896257791E-007, + 0.49986783979004747E-007, 0.51186879246229022E-007, + 0.50009353247048599E-007, 0.48250264542204811E-007, + 0.47560278748093609E-007, 0.48298089720730957E-007, + 0.49095935613468768E-007, 0.49493024126912931E-007, + 0.50250797590476007E-007, 0.51949267668322422E-007, + 0.53778727208416418E-007, 0.53563593301099588E-007, + 0.51218136771199298E-007, 0.43171429694325200E-007, + 0.39019610039033895E-007, 0.36175109143257051E-007, + 0.42731638777892750E-007, 0.38060728507221777E-007, + 0.44046323901481340E-007, 0.39216732751330010E-007, + 0.34842233953609988E-007, 0.34068804733226066E-007, + 0.30636043694263528E-007, 0.28302341686131413E-007, + 0.33023014309036320E-007, 0.34745748365385196E-007, + 0.43623545003583371E-007, 0.48206451795644064E-007, + 0.49854490325455530E-007, 0.50346335647724146E-007, + 0.50661560988561763E-007, 0.50986261962838767E-007, + 0.51256955985111086E-007, 0.51482578449096488E-007, + 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + } + } +} + KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -469,9 +680,8 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), [=](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -559,33 +769,61 @@ void update_cloud_borne_aerosols( } } -// Update interstitial aerosols using tendencies +// Update interstitial aerosols using tendencies - levels +void update_interstitial_aerosols_levs( + const haero::ThreadTeam &team, const int nlev, const int icol, + const int s_idx, const Real dt, + const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + // output + MAMAci::view_2d aero_mr) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [=](int kk) { + aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; + }); +} + +// Update interstitial aerosols using tendencies- cols and levs void update_interstitial_aerosols( haero::ThreadTeamPolicy team_policy, const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], const int nlev, const Real dt, // output mam_coupling::AerosolState &dry_aero) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); + // starting index of ptend_q array (for MAM4, pcnst=40, ncnst_tot=25 ) + int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // loop through all modes and species + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + // mass mixing ratio of species "a" of mode "m" + auto aero_mmr = dry_aero.int_aero_mmr[m][a]; + + if(aero_mmr.data()) { Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), [=](int kk) { - int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - for(int a = 0; a < mam4::num_species_mode(m); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](icol, kk) += - ptend_q[s_idx](icol, kk) * dt; - s_idx++; - } - } - dry_aero.int_aero_nmr[m](icol, kk) += - ptend_q[s_idx](icol, kk) * dt; - s_idx++; - } + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // update values for all levs at this column + update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, + ptend_q, + // output + aero_mmr); }); - }); + // update index for the next species (only if aero_mmr.data() is True) + ++s_idx; + } + } + auto aero_nmr = + dry_aero.int_aero_nmr[m]; // number mixing ratio for mode "m" + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // update values for all levs at this column + update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, + ptend_q, + // output + aero_nmr); + }); + ++s_idx; // update index for the next species + } } void call_hetfrz_compute_tendencies( @@ -801,7 +1039,8 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + // add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -815,7 +1054,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -1182,6 +1421,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { + // FIXME: Remove set_input and print_input + set_input(w_sec_int_, kvh_int_, ncol_, nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -1190,16 +1432,17 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); // Get w_sec_int_ from w_sec_mid_ - compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + /*compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output - w_sec_int_); + w_sec_int_);*/ compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output @@ -1218,6 +1461,8 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + // FIXME:Remove set_dgait + set_dgait(aitken_dry_dia_, ncol_, nlev_); // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is @@ -1243,9 +1488,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. // Get kvh_int_ from kvh_mid_ - compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, - // output - kvh_int_); + // compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + // // output + // kvh_int_); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial @@ -1291,9 +1536,122 @@ void MAMAci::run_impl(const double dt) { // output dry_aero_); + const int kb = 62; + print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), + wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), + rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, + qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), + hetfrz_contact_nucleation_tend_(0, kb), + hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); + // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function } } // namespace scream + +/* FORTRAN output + + AT THE END OF MICROP_AERO---- + ----- OUPUT AT time step: 6 6 + wo 3.343177351577457E-004 + rho: 1.28186433384113 + TKE: 1.666465903934813E-002 6.097664104873541E-002 + wsub: 0.160873967324407 + wsubi: 0.200000000000000 + wsig: 0.160873967324407 + dgnum_ait: 4.362354500358337E-008 + naai: 11464.5957222634 + naai_hom: 29838.6879763933 + factnum: 0.998556176544263 0.861689826773470 + 0.999999999974140 0.000000000000000E+000 + nctend_mixnuc: -839.404918218856 + O3_state: 7.467031959984916E-008 + H2O2_state: 9.349365721024434E-013 + H2SO4_state: 6.828290138182288E-015 + SO2_state: 5.331014679199532E-012 + DMS_state: 2.308803742493950E-012 + SOAG_state: 5.353788713507973E-012 + so4_a1_state: 2.725322171440633E-011 + pom_a1_state: 5.550204151691568E-012 + soa_a1_state: 4.265179209409266E-011 + bc_a1_state: 4.464761667085421E-013 + dst_a1_state: 4.822960238167224E-013 + ncl_a1_state: 1.572123611679146E-013 + mom_a1_state: 1.295791882646146E-014 + num_a1_state: 3113980.27259761 + so4_a2_state: 5.931234261210945E-014 + soa_a2_state: 5.442455780020074E-015 + ncl_a2_state: 1.536147864466575E-015 + mom_a2_state: 1.003300717028814E-016 + num_a2_state: 302994.843261936 + dst_a3_state: 1.680288936900572E-011 + ncl_a3_state: 2.766061699712112E-012 + so4_a3_state: 2.666433567899354E-013 + bc_a3_state: 2.232845976874488E-015 + pom_a3_state: 6.726406544964171E-015 + soa_a3_state: 5.446646976638719E-014 + mom_a3_state: 4.675942808467773E-016 + num_a3_state: 889.148316994625 + pom_a4_state: 2.796218924673357E-012 + bc_a4_state: 2.713331090178062E-013 + mom_a4_state: 1.155299651446152E-018 + num_a4_state: 2110535.03879661 + O3_tend: 0.000000000000000E+000 + H2O2_tend: 0.000000000000000E+000 + H2SO4_tend: 0.000000000000000E+000 + SO2_tend: 0.000000000000000E+000 + DMS_tend: 0.000000000000000E+000 + SOAG_tend: 0.000000000000000E+000 + so4_a1_tend: -1.384310943569190E-014 + pom_a1_tend: -2.950799002682271E-015 + soa_a1_tend: -2.283869371271150E-014 + bc_a1_tend: -2.202672700280516E-016 + dst_a1_tend: -2.194060414083922E-016 + ncl_a1_tend: -6.981124745267083E-017 + mom_a1_tend: -5.662690535048673E-018 + num_a1_tend: -1626.38730532537 + so4_a2_tend: -2.548707408341951E-017 + soa_a2_tend: -1.929990276139139E-018 + ncl_a2_tend: -6.855087738119723E-019 + mom_a2_tend: -4.571691604185304E-020 + num_a2_tend: -111.890330245159 + dst_a3_tend: -7.609611333715698E-015 + ncl_a3_tend: -1.256350252705383E-015 + so4_a3_tend: -1.320988655880598E-016 + bc_a3_tend: -1.099586316024402E-018 + pom_a3_tend: -3.309939677824050E-018 + soa_a3_tend: -2.767668335981830E-017 + mom_a3_tend: -2.254911604324925E-019 + num_a3_tend: -0.406788905791186 + pom_a4_tend: -2.375070578317096E-017 + bc_a4_tend: 4.082479713528180E-017 + mom_a4_tend: -4.901356204764327E-023 + num_a4_tend: -34.4232851017138 + At the end:num_c1 1322558.40616450 + At the end:so4_c1 1.233997159684093E-011 + At the end:pom_c1 2.393397505793018E-012 + At the end:soa_c1 1.872520079842212E-011 + At the end:bc_c1 1.922157644491776E-013 + At the end:dst_c1 2.530385880698310E-013 + At the end:ncl_c1 8.315014074126540E-014 + At the end:mom_c1 6.895143982427237E-015 + At the end:num_c2 142214.467855250 + At the end:so4_c2 3.181118913367564E-014 + At the end:soa_c2 2.996638282752921E-015 + At the end:ncl_c2 8.055333842809433E-016 + At the end:mom_c2 5.202632275847699E-017 + At the end:num_c3 464.524744048298 + At the end:dst_c3 8.814810290885256E-012 + At the end:ncl_c3 1.417415096039318E-012 + At the end:so4_c3 1.284022582681906E-013 + At the end:bc_c3 1.081226492056768E-015 + At the end:pom_c3 3.292708392236485E-015 + At the end:soa_c3 2.584587804061707E-014 + At the end:mom_c3 2.324887367507579E-016 + At the end:num_c4 6.934165370527951E-014 + At the end:pom_c4 2.703924752644086E-026 + At the end:bc_c4 3.684880329580097E-027 + At the end:mom_c4 1.049556651713384E-032 +*/ \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 394ab89f21e..3f6f7a09314 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -155,9 +155,7 @@ class MAMAci final : public scream::AtmosphereProcess { MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // Process metadata: Return type of the process - AtmosphereProcessType type() const { - return AtmosphereProcessType::Physics; - } + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } // Return name of the process std::string name() const { return "mam4_aci"; } diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index c2fcfb6a8ea..fc5077ead49 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -646,11 +646,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -671,21 +671,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index b175e6f5e7c..7d77ba718fe 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,7 +12,8 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +#SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +SetVarDependingOnTestSize(NUM_STEPS 1 1 1) set (ATM_TIME_STEP 600) set (RUN_T0 2021-10-12-45000) diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index 2c4172b0da1..dd164585b6b 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -26,17 +26,12 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file - - #we should get the following variables from other processes - pbl_height : 1.0 - dgnum: 4.362354500358337E-008 - cldfrac_liq_prev: 0.138584624960092 - eddy_diff_heat: 0.620820118789603 - w_variance: 1.110977269289875E-002 + pbl_height: 1.0 + dgnum: 0.1 # The parameters for I/O control Scorpio: From 8f237ffe01a46b02c7ba44cd402f43c27f177744 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 30 Apr 2024 14:00:58 -0700 Subject: [PATCH 110/142] Fixes GPU runtime error about illegal memory access --- .../mam/eamxx_mam_aci_process_interface.cpp | 350 +----------------- .../eamxx/src/physics/mam/mam_coupling.hpp | 20 +- .../single-process/mam/aci/CMakeLists.txt | 5 +- .../tests/single-process/mam/aci/input.yaml | 13 +- 4 files changed, 31 insertions(+), 357 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 9e3c272ef89..64e0e3420e8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -21,217 +21,6 @@ namespace scream { namespace { -void print_output(const Real w0, const Real rho, const Real tke, - const Real wsub, const Real wice, const Real wsig, - const Real naai_hom, const Real naai, const Real rpdel, - MAMAci::view_3d factnum, const Real tendnd, - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - const Real hetfrz_immersion_nucleation_tend, - const Real hetfrz_contact_nucleation_tend, - const Real hetfrz_depostion_nucleation_tend, - const mam_coupling::AerosolState &dry_aero, const int kb) { - std::cout << "w0:" << w0 << std::endl; - std::cout << " rho: " << rho << std::endl; - std::cout << "TKE:" << tke << std::endl; - std::cout << "WSUB:" << wsub << std::endl; - std::cout << "WICE:" << wice << std::endl; - std::cout << "WSIG:" << wsig << std::endl; - std::cout << "naai_hom_:" << naai_hom << std::endl; - std::cout << "naai_:" << naai << std::endl; - std::cout << "rpdel_:" << rpdel << std::endl; - std::cout << "factnum_:" << factnum(0, 0, kb) << " : " << factnum(0, 1, kb) - << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) - << std::endl; - std::cout << "tendnd_:" << tendnd << std::endl; - for(int ic = 9; ic < 40; ++ic) { - std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; - } - for(int ic = 0; ic < 25; ++ic) { - std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) - << std::endl; - } - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend << ":" - << hetfrz_immersion_nucleation_tend << std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" - << hetfrz_contact_nucleation_tend << std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" - << hetfrz_depostion_nucleation_tend << std::endl; -} - -void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, - const int ncol_, const int nlev_) { - const Real w_sec_e3sm[73] = { - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.44201892319518146E-003, - 0.77315620137962326E-003, 0.24806301482800117E-002, - 0.11109772692898754E-001, 0.40651094032490273E-001, - 0.82156694426095800E-001, 0.12207124453993526E+000, - 0.15516728994634110E+000, 0.17775318086169636E+000, - 0.18549817250146838E+000, 0.17184548286554119E+000, - 0.12741230682196053E+000, 0.65495229516041628E-001, - 0.26909155217660592E-001}; - - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_e3sm[kk]; - // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); - } - // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); - w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; - } - const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, - 0.75991761081225006E-002, 0.12291092068185365E-001, - 0.11484807652762415E-001, 0.10856880396302943E-001, - 0.10500384508819637E-001, 0.10361486171738229E-001, - 0.10333325067964508E-001, 0.10258838031435397E-001, - 0.10027325248446619E-001, 0.97784259072973521E-002, - 0.96611845055866539E-002, 0.96249746122327937E-002, - 0.95773431515696512E-002, 0.95180614513688099E-002, - 0.94713233348487150E-002, 0.94503864489758338E-002, - 0.94536294366578833E-002, 0.94575972194308883E-002, - 0.94403767489615684E-002, 0.93975694769176284E-002, - 0.93322843554751022E-002, 0.92777070192527501E-002, - 0.92456776697171228E-002, 0.92266924824142716E-002, - 0.92123025773060436E-002, 0.91888715633294191E-002, - 0.91516797753615851E-002, 0.90958299606649744E-002, - 0.89988037524983237E-002, 0.88220984587642423E-002, - 0.85231270833157156E-002, 0.81397522619395188E-002, - 0.79160421807845088E-002, 0.81206851117902653E-002, - 0.86526891616674779E-002, 0.91682975412125615E-002, - 0.96043394254592580E-002, 0.10033689085881327E-001, - 0.10428656694074272E-001, 0.10715913043864789E-001, - 0.10919631245454951E-001, 0.11250937075285789E-001, - 0.11829292157343831E-001, 0.12413311776454055E-001, - 0.12851317662157077E-001, 0.13175523677700330E-001, - 0.13224182907540188E-001, 0.13085937680733115E-001, - 0.12615055546741534E-001, 0.11995423733019836E-001, - 0.12346556881757400E-001, 0.13433752971524651E-001, - 0.13904308240950175E-001, 0.13539811748121957E-001, - 0.12555099320041433E-001, 0.11519643673351362E-001, - 0.11414071302852231E-001, 0.13409756835238139E-001, - 0.24071962815959351E-001, 0.75489419450816414E-001, - 0.62082011878960308E+000, 0.63952862312816796E+001, - 0.16226857944175123E+002, 0.21882852534279891E+002, - 0.24966173574402408E+002, 0.25710753126453692E+002, - 0.24069881024271943E+002, 0.19743922403487922E+002, - 0.98667814246712027E+001, 0.25633359450143991E+001, - 0.14682471685037493E+001}; - // compute eddy diffusivity of heat at the interfaces - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - // kvh_int_(icol, kk) = kvh_mid_(icol, kk); - kvh_int_(icol, kk) = kvh_e3sm[kk]; - } - // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); - kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; - } -} -void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, - const int nlev_) { - const Real dgnum_ait_e3sm[72] = { - 0.20877713336487552E-007, 0.21782230353342090E-007, - 0.21688324003865861E-007, 0.21112855042342451E-007, - 0.19162058462939536E-007, 0.18102979880838476E-007, - 0.17906980715477606E-007, 0.20271254074583327E-007, - 0.22698983422181942E-007, 0.24134835117044986E-007, - 0.25498156808001372E-007, 0.29796738799905547E-007, - 0.35822987394021908E-007, 0.41170963764365215E-007, - 0.44892726528330642E-007, 0.47217231342203107E-007, - 0.48928661807108766E-007, 0.50170939816128735E-007, - 0.51078750853732200E-007, 0.52247333465736065E-007, - 0.53190758580174931E-007, 0.53576491941850044E-007, - 0.53915614473890715E-007, 0.54510964775236826E-007, - 0.55643231691556703E-007, 0.57057811112589899E-007, - 0.58177383586181116E-007, 0.58209849180850108E-007, - 0.57976751598840998E-007, 0.52000000000000002E-007, - 0.50728746567226150E-007, 0.49119902704480870E-007, - 0.48212162162050883E-007, 0.49227715213506454E-007, - 0.46876827233752246E-007, 0.45360603896257791E-007, - 0.49986783979004747E-007, 0.51186879246229022E-007, - 0.50009353247048599E-007, 0.48250264542204811E-007, - 0.47560278748093609E-007, 0.48298089720730957E-007, - 0.49095935613468768E-007, 0.49493024126912931E-007, - 0.50250797590476007E-007, 0.51949267668322422E-007, - 0.53778727208416418E-007, 0.53563593301099588E-007, - 0.51218136771199298E-007, 0.43171429694325200E-007, - 0.39019610039033895E-007, 0.36175109143257051E-007, - 0.42731638777892750E-007, 0.38060728507221777E-007, - 0.44046323901481340E-007, 0.39216732751330010E-007, - 0.34842233953609988E-007, 0.34068804733226066E-007, - 0.30636043694263528E-007, 0.28302341686131413E-007, - 0.33023014309036320E-007, 0.34745748365385196E-007, - 0.43623545003583371E-007, 0.48206451795644064E-007, - 0.49854490325455530E-007, 0.50346335647724146E-007, - 0.50661560988561763E-007, 0.50986261962838767E-007, - 0.51256955985111086E-007, 0.51482578449096488E-007, - 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - } - } -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -680,8 +469,9 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), [=](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -1039,8 +829,7 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - // add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); - add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -1054,7 +843,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -1421,9 +1210,6 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - // FIXME: Remove set_input and print_input - set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -1432,17 +1218,16 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); // Get w_sec_int_ from w_sec_mid_ - /*compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output - w_sec_int_);*/ + w_sec_int_); compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output @@ -1461,8 +1246,6 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - // FIXME:Remove set_dgait - set_dgait(aitken_dry_dia_, ncol_, nlev_); // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is @@ -1488,9 +1271,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. // Get kvh_int_ from kvh_mid_ - // compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, - // // output - // kvh_int_); + compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + // output + kvh_int_); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial @@ -1536,122 +1319,9 @@ void MAMAci::run_impl(const double dt) { // output dry_aero_); - const int kb = 62; - print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), - wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), - rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, - qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), - hetfrz_contact_nucleation_tend_(0, kb), - hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); - // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function } } // namespace scream - -/* FORTRAN output - - AT THE END OF MICROP_AERO---- - ----- OUPUT AT time step: 6 6 - wo 3.343177351577457E-004 - rho: 1.28186433384113 - TKE: 1.666465903934813E-002 6.097664104873541E-002 - wsub: 0.160873967324407 - wsubi: 0.200000000000000 - wsig: 0.160873967324407 - dgnum_ait: 4.362354500358337E-008 - naai: 11464.5957222634 - naai_hom: 29838.6879763933 - factnum: 0.998556176544263 0.861689826773470 - 0.999999999974140 0.000000000000000E+000 - nctend_mixnuc: -839.404918218856 - O3_state: 7.467031959984916E-008 - H2O2_state: 9.349365721024434E-013 - H2SO4_state: 6.828290138182288E-015 - SO2_state: 5.331014679199532E-012 - DMS_state: 2.308803742493950E-012 - SOAG_state: 5.353788713507973E-012 - so4_a1_state: 2.725322171440633E-011 - pom_a1_state: 5.550204151691568E-012 - soa_a1_state: 4.265179209409266E-011 - bc_a1_state: 4.464761667085421E-013 - dst_a1_state: 4.822960238167224E-013 - ncl_a1_state: 1.572123611679146E-013 - mom_a1_state: 1.295791882646146E-014 - num_a1_state: 3113980.27259761 - so4_a2_state: 5.931234261210945E-014 - soa_a2_state: 5.442455780020074E-015 - ncl_a2_state: 1.536147864466575E-015 - mom_a2_state: 1.003300717028814E-016 - num_a2_state: 302994.843261936 - dst_a3_state: 1.680288936900572E-011 - ncl_a3_state: 2.766061699712112E-012 - so4_a3_state: 2.666433567899354E-013 - bc_a3_state: 2.232845976874488E-015 - pom_a3_state: 6.726406544964171E-015 - soa_a3_state: 5.446646976638719E-014 - mom_a3_state: 4.675942808467773E-016 - num_a3_state: 889.148316994625 - pom_a4_state: 2.796218924673357E-012 - bc_a4_state: 2.713331090178062E-013 - mom_a4_state: 1.155299651446152E-018 - num_a4_state: 2110535.03879661 - O3_tend: 0.000000000000000E+000 - H2O2_tend: 0.000000000000000E+000 - H2SO4_tend: 0.000000000000000E+000 - SO2_tend: 0.000000000000000E+000 - DMS_tend: 0.000000000000000E+000 - SOAG_tend: 0.000000000000000E+000 - so4_a1_tend: -1.384310943569190E-014 - pom_a1_tend: -2.950799002682271E-015 - soa_a1_tend: -2.283869371271150E-014 - bc_a1_tend: -2.202672700280516E-016 - dst_a1_tend: -2.194060414083922E-016 - ncl_a1_tend: -6.981124745267083E-017 - mom_a1_tend: -5.662690535048673E-018 - num_a1_tend: -1626.38730532537 - so4_a2_tend: -2.548707408341951E-017 - soa_a2_tend: -1.929990276139139E-018 - ncl_a2_tend: -6.855087738119723E-019 - mom_a2_tend: -4.571691604185304E-020 - num_a2_tend: -111.890330245159 - dst_a3_tend: -7.609611333715698E-015 - ncl_a3_tend: -1.256350252705383E-015 - so4_a3_tend: -1.320988655880598E-016 - bc_a3_tend: -1.099586316024402E-018 - pom_a3_tend: -3.309939677824050E-018 - soa_a3_tend: -2.767668335981830E-017 - mom_a3_tend: -2.254911604324925E-019 - num_a3_tend: -0.406788905791186 - pom_a4_tend: -2.375070578317096E-017 - bc_a4_tend: 4.082479713528180E-017 - mom_a4_tend: -4.901356204764327E-023 - num_a4_tend: -34.4232851017138 - At the end:num_c1 1322558.40616450 - At the end:so4_c1 1.233997159684093E-011 - At the end:pom_c1 2.393397505793018E-012 - At the end:soa_c1 1.872520079842212E-011 - At the end:bc_c1 1.922157644491776E-013 - At the end:dst_c1 2.530385880698310E-013 - At the end:ncl_c1 8.315014074126540E-014 - At the end:mom_c1 6.895143982427237E-015 - At the end:num_c2 142214.467855250 - At the end:so4_c2 3.181118913367564E-014 - At the end:soa_c2 2.996638282752921E-015 - At the end:ncl_c2 8.055333842809433E-016 - At the end:mom_c2 5.202632275847699E-017 - At the end:num_c3 464.524744048298 - At the end:dst_c3 8.814810290885256E-012 - At the end:ncl_c3 1.417415096039318E-012 - At the end:so4_c3 1.284022582681906E-013 - At the end:bc_c3 1.081226492056768E-015 - At the end:pom_c3 3.292708392236485E-015 - At the end:soa_c3 2.584587804061707E-014 - At the end:mom_c3 2.324887367507579E-016 - At the end:num_c4 6.934165370527951E-014 - At the end:pom_c4 2.703924752644086E-026 - At the end:bc_c4 3.684880329580097E-027 - At the end:mom_c4 1.049556651713384E-032 -*/ \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index fc5077ead49..c2fcfb6a8ea 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -646,11 +646,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -671,21 +671,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 7d77ba718fe..8c5028828bb 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,8 +12,7 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -#SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) set (ATM_TIME_STEP 600) set (RUN_T0 2021-10-12-45000) @@ -43,4 +42,4 @@ if (SCREAM_ENABLE_BASELINE_TESTS) # Note: one is enough, since we already check that np1 is BFB with npX set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) -endif() \ No newline at end of file +endif() diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index dd164585b6b..fe26d828d41 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -26,14 +26,19 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file - pbl_height: 1.0 - dgnum: 0.1 + + #we should get the following variables from other processes + pbl_height : 1.0 + dgnum: 4.362354500358337E-008 + cldfrac_liq_prev: 0.138584624960092 + eddy_diff_heat: 0.620820118789603 + w_variance: 1.110977269289875E-002 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] -... \ No newline at end of file +... From 6c77406b20370449de6f082cf661c98288605f0f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 30 Apr 2024 14:09:49 -0700 Subject: [PATCH 111/142] Clang format --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 64e0e3420e8..b9c8ae9e8f8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -469,9 +469,8 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), [=](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) From c047515ad32f6da84fb75ccf172239f77d6e2c57 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 30 Apr 2024 20:18:18 -0700 Subject: [PATCH 112/142] After rebase changes, adds make_layout for aci --- .../mam/eamxx_mam_aci_process_interface.cpp | 10 +++++-- .../eamxx/src/physics/mam/mam_coupling.hpp | 29 ------------------- .../eamxx/src/share/field/field_tag.hpp | 28 ------------------ 3 files changed, 8 insertions(+), 59 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b9c8ae9e8f8..33d57f2050d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -771,6 +771,12 @@ void MAMAci::set_grids( // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + auto make_layout = [](const std::vector &extents, + const std::vector &names) { + std::vector tags(extents.size(), CMP); + return FieldLayout(tags, extents, names); + }; + using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); @@ -846,8 +852,8 @@ void MAMAci::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{{COL, NMODES, LEV}, - {ncol_, num_aero_modes, nlev_}}; + FieldLayout scalar4d_layout_mid = + make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); // dry diameter of aerosols [m] add_field("dgnum", scalar4d_layout_mid, m, grid_name); diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index c2fcfb6a8ea..ae8c84210b5 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -836,19 +836,9 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { -<<<<<<< HEAD - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); -======= int m = static_cast(mode_index); -<<<<<<< HEAD - if(aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); ->>>>>>> Modified species order to match E3SM with the folling change in mam4xx -======= if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const Real mw = mam4::aero_species(a).molecular_weight; vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); @@ -879,19 +869,9 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { -<<<<<<< HEAD - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); -======= int m = static_cast(mode_index); -<<<<<<< HEAD - if(aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); ->>>>>>> Modified species order to match E3SM with the folling change in mam4xx -======= if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const Real mw = mam4::aero_species(a).molecular_weight; q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); @@ -927,17 +907,8 @@ void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], int a = aerosol_index_for_mode(mode_index, aero_id); progs.q_aero_i[m][a](k) = q[i]; progs.q_aero_c[m][a](k) = qqcw[i]; -<<<<<<< HEAD -<<<<<<< HEAD - } else { // constituent is a modal number mixing ratio -======= - } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); ->>>>>>> Modified species order to match E3SM with the folling change in mam4xx -======= } else { // constituent is a modal number mixing ratio int m = static_cast(mode_index); ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index progs.n_mode_i[m](k) = q[i]; progs.n_mode_c[m](k) = qqcw[i]; } diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index a2e19662162..673208d6ae8 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -34,12 +34,6 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, - // - MAM_NumModes, - MAM_NumRefIndexReal, - MAM_NumRefIndexImag, - MAM_NumCoefficients, - MAM_NumModesInFile }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -56,13 +50,6 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; - constexpr auto NMODES = FieldTag::MAM_NumModes; - // - constexpr auto NREFINDEX_REAL = FieldTag::MAM_NumRefIndexReal; - constexpr auto NREFINDEX_IM = FieldTag::MAM_NumRefIndexImag; - - constexpr auto NCOEF_NUMBER = FieldTag::MAM_NumCoefficients; - constexpr auto MODE = FieldTag::MAM_NumModesInFile; } inline std::string e2str (const FieldTag ft) { @@ -93,21 +80,6 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; - case FieldTag::MAM_NumModes: - name = "num_modes"; - break; - case FieldTag::MAM_NumRefIndexReal: - name = "refindex_real"; - break; - case FieldTag::MAM_NumRefIndexImag: - name = "refindex_im"; - break; - case FieldTag::MAM_NumCoefficients: - name = "coef_number"; - break; - case FieldTag::MAM_NumModesInFile: - name = "mode"; - break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } From f7d3e404a77746247192a6162dee28b429630aca Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 1 May 2024 08:53:31 -0700 Subject: [PATCH 113/142] Fixes wrong layouts and removed function to copy views --- .../eamxx/src/physics/shoc/CMakeLists.txt | 1 - .../shoc/disp/shoc_assign_2d_view_disp.cpp | 29 ------------------- .../shoc/eamxx_shoc_process_interface.cpp | 22 +++++++------- .../src/physics/shoc/impl/shoc_main_impl.hpp | 6 ++-- .../eamxx/src/physics/shoc/shoc_functions.hpp | 7 ----- 5 files changed, 14 insertions(+), 51 deletions(-) delete mode 100644 components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index f57506fe0ed..e37379095d0 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -85,7 +85,6 @@ set(SHOC_SK_SRCS disp/shoc_diag_third_shoc_moments_disp.cpp disp/shoc_assumed_pdf_disp.cpp disp/shoc_update_host_dse_disp.cpp - disp/shoc_assign_2d_view_disp.cpp ) if (NOT SCREAM_DEBUG) diff --git a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp deleted file mode 100644 index e6cd04054d0..00000000000 --- a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "shoc_functions.hpp" - -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream { -namespace shoc { - -template<> -void Functions -::shoc_assign_2d_view_disp( - const Int& shcol, - const Int& nlev, - const view_2d& input_view, - const view_2d& output_view) -{ - using ExeSpace = typename KT::ExeSpace; - - const auto nlev_packs = ekat::npack(nlev); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nlev_packs); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - const Int i = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { - output_view (i,k) = input_view(i,k); - }); - }); -} - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 25c2674c912..5ceb40c738f 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -80,19 +80,19 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d_layout_col, m, grid_name); - add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); - add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("w_variance", scalar3d_layout_mid, m2/s2, grid_name, ps); - add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("pbl_height", scalar2d , m, grid_name); + add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); + add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); + add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); + add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index ca163a49ddd..640400f54d2 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -575,7 +575,7 @@ void Functions::shoc_main_internal( pblh); // Output // Assign tkh to the tkh output variable - shoc_assign_2d_view_disp(shcol, nlev, tkh_out,tkh); + Kokkos::deep_copy(tkh_out,tkh); } #endif @@ -654,7 +654,7 @@ Int Functions::shoc_main( const auto shoc_cldfrac_s = ekat::subview(shoc_input_output.shoc_cldfrac, i); const auto shoc_ql_s = ekat::subview(shoc_input_output.shoc_ql, i); const auto shoc_ql2_s = ekat::subview(shoc_output.shoc_ql2, i); - const auto tkh_out_s = ekat::subview(shoc_output.tkh, i); + const auto tkh_out_s = ekat::subview(shoc_output.tkh, i); const auto shoc_mix_s = ekat::subview(shoc_history_output.shoc_mix, i); const auto w_sec_s = ekat::subview(shoc_history_output.w_sec, i); const auto thl_sec_s = ekat::subview(shoc_history_output.thl_sec, i); @@ -686,7 +686,7 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_out_s, // Output + pblh_s, shoc_ql2_s, tkh_out_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 0b651263064..ccc941bfd05 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -1205,13 +1205,6 @@ struct Functions const view_2d& tk, const view_2d& tkh, const view_2d& isotropy); - - static void shoc_assign_2d_view_disp( - const Int& shcol, - const Int& nlev, - const view_2d& input_view, - const view_2d& output_view); - #endif }; // struct Functions From 5e90223fbadb3779b2860f696bc899ec6ff605d0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 2 May 2024 17:27:23 -0700 Subject: [PATCH 114/142] Fixes process order by adding spa, adds dgnum nominal value --- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 ++ .../testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands | 1 + 2 files changed, 3 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 49244880280..1942e339ef7 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -487,6 +487,8 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0,0.0 + + 2.6e-08 0.0 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index 3bffab45ba6..6ce4e74690c 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -4,6 +4,7 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp" -b $CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b From 99782e4d0950e113a6a20610a8da87c3acfd1450 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 2 May 2024 21:07:22 -0700 Subject: [PATCH 115/142] Explicitly specify do_predict_nc and do_prescribed_ccn flags --- .../testmods_dirs/scream/mam4xx/aci/shell_commands | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index 6ce4e74690c..68eaf51698b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -3,8 +3,21 @@ #Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options ./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#modify initial condition file to get aerosol species ICs $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b + +# Add spa as RRTMG needs spa $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp" -b + +# Replace spa with mam4_aci to invoke mam4 aci scheme $CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b +#Set precribed ccn to false so that P3 uses input from ACI +$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_prescribed_ccn=false -b + +#Set predicted ccn to true so that P3 uses input from ACI +$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_predict_nc=true -b + + + From 2fcc3c319da01adb4fe7f1c14d8e4fc683530846 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 4 May 2024 12:24:46 -0700 Subject: [PATCH 116/142] Temporarily adds cloudborne aerosols in IC section of the namelist --- .../cime_config/namelist_defaults_scream.xml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 1942e339ef7..50019833d8a 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -489,6 +489,31 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0,0.0 2.6e-08 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 0.0 From efa21196a077000eed27704abd14fc86c1215385 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 6 May 2024 12:27:15 -0600 Subject: [PATCH 117/142] Fix allocation of tkh. --- .../eamxx/src/physics/shoc/shoc_functions_f90.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index 0e4acceb1a7..20fdb7b3ee8 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2743,8 +2743,6 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* qw_sec, Real* qwthl_sec, Real* wthl_sec, Real* wqw_sec, Real* wtke_sec, Real* uw_sec, Real* vw_sec, Real* w3, Real* wqls_sec, Real* brunt, Real* shoc_ql2) { - // tkh is a local variable in C++ impl - (void)tkh; using SHF = Functions; @@ -2758,7 +2756,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, // Initialize Kokkos views, sync to device static constexpr Int num_1d_arrays = 7; - static constexpr Int num_2d_arrays = 34; + static constexpr Int num_2d_arrays = 35; static constexpr Int num_3d_arrays = 1; std::vector temp_1d_d(num_1d_arrays); @@ -2768,14 +2766,14 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, std::vector dim1_2d_sizes = {shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, - shcol, shcol, shcol, shcol, + shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol}; std::vector dim2_2d_sizes = {nlev, nlevi, nlev, nlevi, nlev, nlev, nlev, num_qtracers, nlev, nlev, nlev, nlev, nlev, nlev, nlev, - nlev, nlev, nlev, nlev, + nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev, nlev, nlev}; @@ -2786,7 +2784,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, thv, w_field, wtracer_sfc, inv_exner, host_dse, tke, thetal, qw, u_wind, v_wind, wthv_sec, tk, shoc_cldfrac, shoc_ql, - shoc_ql2, shoc_mix, w_sec, thl_sec, qw_sec, + shoc_ql2, tkh, shoc_mix, w_sec, thl_sec, qw_sec, qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, vw_sec, w3, wqls_sec, brunt, isotropy}; @@ -2827,6 +2825,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, shoc_cldfrac_d(temp_2d_d[index_counter++]), shoc_ql_d (temp_2d_d[index_counter++]), shoc_ql2_d (temp_2d_d[index_counter++]), + tkh_d (temp_2d_d[index_counter++]), shoc_mix_d (temp_2d_d[index_counter++]), w_sec_d (temp_2d_d[index_counter++]), thl_sec_d (temp_2d_d[index_counter++]), @@ -2879,7 +2878,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, SHF::SHOCInputOutput shoc_input_output{host_dse_d, tke_d, thetal_d, qw_d, horiz_wind_d, wthv_sec_d, qtracers_cxx_d, tk_d, shoc_cldfrac_d, shoc_ql_d}; - SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d}; + SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d, tkh_d}; SHF::SHOCHistoryOutput shoc_history_output{shoc_mix_d, w_sec_d, thl_sec_d, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, From 893df5e0dd15efd2622af679991503e1db27da66 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 6 May 2024 13:13:41 -0600 Subject: [PATCH 118/142] Seems that tkh is an array of nlev+1, not nlev. --- .../src/physics/shoc/shoc_functions_f90.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index 20fdb7b3ee8..7e3ae283418 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2774,19 +2774,19 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, nlev, nlev, num_qtracers, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, - nlev, nlev, nlev, nlevi, nlevi, + nlevi, nlev, nlev, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev, nlev, nlev}; std::vector ptr_array_1d = {host_dx, host_dy, wthl_sfc, wqw_sfc, uw_sfc, vw_sfc, phis}; - std::vector ptr_array_2d = {zt_grid, zi_grid, pres, presi, pdel, - thv, w_field, wtracer_sfc, inv_exner, host_dse, - tke, thetal, qw, u_wind, v_wind, - wthv_sec, tk, shoc_cldfrac, shoc_ql, - shoc_ql2, tkh, shoc_mix, w_sec, thl_sec, qw_sec, - qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, - vw_sec, w3, wqls_sec, brunt, isotropy}; + std::vector ptr_array_2d = {zt_grid, zi_grid, pres, presi, pdel, + thv, w_field, wtracer_sfc, inv_exner, host_dse, + tke, thetal, qw, u_wind, v_wind, + wthv_sec, tk, shoc_cldfrac, shoc_ql, shoc_ql2, + tkh, shoc_mix, w_sec, thl_sec, qw_sec, + qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, + vw_sec, w3, wqls_sec, brunt, isotropy}; ScreamDeepCopy::copy_to_device(ptr_array_1d, shcol, temp_1d_d); ekat::host_to_device(ptr_array_2d, dim1_2d_sizes, dim2_2d_sizes, temp_2d_d, true); From 04bd1a6415774be36c8cc82213a135785fb76cdb Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 6 May 2024 15:01:00 -0600 Subject: [PATCH 119/142] Use this version if tkh is nlev in length, not nlev+1. --- components/eamxx/src/physics/shoc/shoc_functions_f90.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index 7e3ae283418..5624169ccfb 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2774,7 +2774,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, nlev, nlev, num_qtracers, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, - nlevi, nlev, nlev, nlevi, nlevi, + nlev, nlev, nlev, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev, nlev, nlev}; @@ -2784,7 +2784,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, thv, w_field, wtracer_sfc, inv_exner, host_dse, tke, thetal, qw, u_wind, v_wind, wthv_sec, tk, shoc_cldfrac, shoc_ql, shoc_ql2, - tkh, shoc_mix, w_sec, thl_sec, qw_sec, + tkh, shoc_mix, w_sec, thl_sec, qw_sec, qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, vw_sec, w3, wqls_sec, brunt, isotropy}; @@ -2909,7 +2909,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, tabs ("shoc_tabs", shcol, nlev_packs), dz_zt ("dz_zt", shcol, nlevi_packs), dz_zi ("dz_zi", shcol, nlevi_packs), - tkhv ("tkh", shcol, nlevi_packs); + tkhv ("tkh", shcol, nlev_packs); SHF::SHOCTemporaries shoc_temporaries{ se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, From cec5469f9f611a3699af4e75b515d6f6106dfa95 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 10 May 2024 08:51:55 -0700 Subject: [PATCH 120/142] Init all computed vars in aci to zero; remove tkh_out from SHOC --- .../mam/eamxx_mam_aci_process_interface.cpp | 36 ++++++++++++++++++- .../shoc/eamxx_shoc_process_interface.cpp | 3 +- .../src/physics/shoc/impl/shoc_main_impl.hpp | 34 +++++++----------- .../eamxx/src/physics/shoc/shoc_functions.hpp | 7 ++-- .../src/physics/shoc/shoc_functions_f90.cpp | 5 ++- 5 files changed, 53 insertions(+), 32 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 33d57f2050d..51c4b9601d8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -189,6 +189,14 @@ void compute_nucleate_ice_tendencies( // from ice nucleation //------------------------------------------------------------- + //Initialize (zero out) all the output + Kokkos::deep_copy(naai, 0.0); + Kokkos::deep_copy(nihf, 0.0); + Kokkos::deep_copy(niim, 0.0); + Kokkos::deep_copy(nidep, 0.0); + Kokkos::deep_copy(nimey, 0.0); + Kokkos::deep_copy(naai_hom, 0.0); + Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -326,10 +334,11 @@ void call_function_dropmixnuc( MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - // ## outputs to be used by other processes ## + // ## input-outputs to be used by other processes ## // qqcw_fld_work should be directly assigned to the cloud borne aerosols MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + // ## outputs to be used by other processes ## // ptend_q are the tendencies to the interstitial aerosols MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], @@ -344,6 +353,26 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { + + + // initialize the purely output variables to zero + for(int i = 0; i < mam4::aero_model::pcnst; ++i)Kokkos::deep_copy(ptend_q[i], 0.0); + Kokkos::deep_copy(factnum, 0.0); + Kokkos::deep_copy(tendnd, 0.0); + + //initialize all diagnostic ouputs to zero: + for(int i = 0; i < mam4::aero_model::pcnst; ++i){ + Kokkos::deep_copy(coltend[i], 0.0); + Kokkos::deep_copy(coltend_cw[i], 0.0); + } + Kokkos::deep_copy(qcld, 0.0); + Kokkos::deep_copy(ndropcol, 0.0); + Kokkos::deep_copy(ndropmix, 0.0); + Kokkos::deep_copy(nsource, 0.0); + Kokkos::deep_copy(wtke, 0.0); + Kokkos::deep_copy(ccn, 0.0); + + // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -1072,10 +1101,15 @@ void MAMAci::initialize_impl(const RunType run_type) { // hetrozenous freezing outputs hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); + Kokkos::deep_copy(hetfrz_immersion_nucleation_tend_, 0.0); //init to zero + hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); + Kokkos::deep_copy(hetfrz_contact_nucleation_tend_, 0.0); //init to zero + hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + Kokkos::deep_copy(hetfrz_depostion_nucleation_tend_, 0.0); //init to zero //--------------------------------------------------------------------------------- // Allocate memory for the class members diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 5ceb40c738f..904d7f9672e 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -186,7 +186,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.inv_exner, &m_buffer.thlm, &m_buffer.qw, &m_buffer.dse, &m_buffer.tke_copy, &m_buffer.qc_copy, &m_buffer.shoc_ql2, &m_buffer.shoc_mix, &m_buffer.isotropy, &m_buffer.w_sec, &m_buffer.wqls_sec, &m_buffer.brunt #ifdef SCREAM_SMALL_KERNELS - , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.tabs, &m_buffer.dz_zt, &m_buffer.tkh + , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.tabs, &m_buffer.dz_zt #endif }; @@ -374,7 +374,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) temporaries.tabs = m_buffer.tabs; temporaries.dz_zt = m_buffer.dz_zt; temporaries.dz_zi = m_buffer.dz_zi; - temporaries.tkh = m_buffer.tkh; #endif shoc_postprocess.set_variables(m_num_cols,m_num_levs,m_num_tracers, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 640400f54d2..120ed505f26 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -119,7 +119,7 @@ void Functions::shoc_main_internal( // Output Variables Scalar& pblh, const uview_1d& shoc_ql2, - const uview_1d& tkh_out, + const uview_1d& tkh, // Diagnostic Output Variables const uview_1d& shoc_mix, const uview_1d& w_sec, @@ -138,10 +138,10 @@ void Functions::shoc_main_internal( { // Define temporary variables - uview_1d rho_zt, shoc_qv, shoc_tabs, dz_zt, dz_zi, tkh; - workspace.template take_many_and_reset<6>( - {"rho_zt", "shoc_qv", "shoc_tabs", "dz_zt", "dz_zi", "tkh"}, - {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi, &tkh}); + uview_1d rho_zt, shoc_qv, shoc_tabs, dz_zt, dz_zi; + workspace.template take_many_and_reset<5>( + {"rho_zt", "shoc_qv", "shoc_tabs", "dz_zt", "dz_zi"}, + {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); // Local scalars Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, @@ -311,15 +311,9 @@ void Functions::shoc_main_internal( workspace, // Workspace pblh); // Output - // Assign tkh to the tkh output variable - const Int nlev_pack = ekat::npack(nlev); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { - tkh_out(k) = tkh(k); - }); - // Release temporary variables from the workspace - workspace.template release_many_contiguous<6>( - {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi, &tkh}); + workspace.template release_many_contiguous<5>( + {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); } #else template @@ -378,7 +372,7 @@ void Functions::shoc_main_internal( // Output Variables const view_1d& pblh, const view_2d& shoc_ql2, - const view_2d& tkh_out, + const view_2d& tkh, // Diagnostic Output Variables const view_2d& shoc_mix, const view_2d& w_sec, @@ -412,8 +406,7 @@ void Functions::shoc_main_internal( const view_2d& shoc_qv, const view_2d& shoc_tabs, const view_2d& dz_zt, - const view_2d& dz_zi, - const view_2d& tkh) + const view_2d& dz_zi) { // Scalarize some views for single entry access const auto s_thetal = ekat::scalarize(thetal); @@ -573,9 +566,6 @@ void Functions::shoc_main_internal( kbfs,shoc_cldfrac, // Input workspace_mgr, // Workspace mgr pblh); // Output - - // Assign tkh to the tkh output variable - Kokkos::deep_copy(tkh_out,tkh); } #endif @@ -654,7 +644,7 @@ Int Functions::shoc_main( const auto shoc_cldfrac_s = ekat::subview(shoc_input_output.shoc_cldfrac, i); const auto shoc_ql_s = ekat::subview(shoc_input_output.shoc_ql, i); const auto shoc_ql2_s = ekat::subview(shoc_output.shoc_ql2, i); - const auto tkh_out_s = ekat::subview(shoc_output.tkh, i); + const auto tkh_s = ekat::subview(shoc_output.tkh, i); const auto shoc_mix_s = ekat::subview(shoc_history_output.shoc_mix, i); const auto w_sec_s = ekat::subview(shoc_history_output.w_sec, i); const auto thl_sec_s = ekat::subview(shoc_history_output.thl_sec, i); @@ -686,7 +676,7 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_out_s, // Output + pblh_s, shoc_ql2_s, tkh_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables @@ -719,7 +709,7 @@ Int Functions::shoc_main( shoc_temporaries.se_a, shoc_temporaries.ke_a, shoc_temporaries.wv_a, shoc_temporaries.wl_a, shoc_temporaries.ustar, shoc_temporaries.kbfs, shoc_temporaries.obklen, shoc_temporaries.ustar2, shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, - shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi, shoc_temporaries.tkh); + shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi); #endif auto finish = std::chrono::steady_clock::now(); diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index ccc941bfd05..74602c688c2 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -890,7 +890,7 @@ struct Functions // Output Variables Scalar& pblh, const uview_1d& shoc_ql2, - const uview_1d& tkh_out, + const uview_1d& tkh, // Diagnostic Output Variables const uview_1d& shoc_mix, const uview_1d& w_sec, @@ -962,7 +962,7 @@ struct Functions // Output Variables const view_1d& pblh, const view_2d& shoc_ql2, - const view_2d& tkh_out, + const view_2d& tkh, // Diagnostic Output Variables const view_2d& shoc_mix, const view_2d& w_sec, @@ -996,8 +996,7 @@ struct Functions const view_2d& shoc_qv, const view_2d& tabs, const view_2d& dz_zt, - const view_2d& dz_zi, - const view_2d& tkh); + const view_2d& dz_zi); #endif // Return microseconds elapsed diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index 5624169ccfb..dc45e2456aa 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2908,12 +2908,11 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, shoc_qv ("shoc_qv", shcol, nlevi_packs), tabs ("shoc_tabs", shcol, nlev_packs), dz_zt ("dz_zt", shcol, nlevi_packs), - dz_zi ("dz_zi", shcol, nlevi_packs), - tkhv ("tkh", shcol, nlev_packs); + dz_zi ("dz_zi", shcol, nlevi_packs); SHF::SHOCTemporaries shoc_temporaries{ se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, - rho_zt, shoc_qv, tabs, dz_zt, dz_zi, tkhv}; + rho_zt, shoc_qv, tabs, dz_zt, dz_zi}; #endif // Create local workspace From c162ee4b31dd9a880e8fb42042a79de2f3e5871c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 10 May 2024 11:10:24 -0700 Subject: [PATCH 121/142] Removes some unneeded deep copies --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 51c4b9601d8..ca7133d8f3a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1101,15 +1101,12 @@ void MAMAci::initialize_impl(const RunType run_type) { // hetrozenous freezing outputs hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); - Kokkos::deep_copy(hetfrz_immersion_nucleation_tend_, 0.0); //init to zero hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - Kokkos::deep_copy(hetfrz_contact_nucleation_tend_, 0.0); //init to zero hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); - Kokkos::deep_copy(hetfrz_depostion_nucleation_tend_, 0.0); //init to zero //--------------------------------------------------------------------------------- // Allocate memory for the class members From 6349d140cb2eeb50ea26a3b05f65b164971a6204 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 12 May 2024 23:40:24 -0700 Subject: [PATCH 122/142] Adds ndropmix and nsource as output variables --- .../mam/eamxx_mam_aci_process_interface.cpp | 50 ++++++------------- 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ca7133d8f3a..5016c8bee82 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -189,14 +189,6 @@ void compute_nucleate_ice_tendencies( // from ice nucleation //------------------------------------------------------------- - //Initialize (zero out) all the output - Kokkos::deep_copy(naai, 0.0); - Kokkos::deep_copy(nihf, 0.0); - Kokkos::deep_copy(niim, 0.0); - Kokkos::deep_copy(nidep, 0.0); - Kokkos::deep_copy(nimey, 0.0); - Kokkos::deep_copy(naai_hom, 0.0); - Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -334,11 +326,10 @@ void call_function_dropmixnuc( MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - // ## input-outputs to be used by other processes ## + // ## outputs to be used by other processes ## // qqcw_fld_work should be directly assigned to the cloud borne aerosols MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - // ## outputs to be used by other processes ## // ptend_q are the tendencies to the interstitial aerosols MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], @@ -353,26 +344,6 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { - - - // initialize the purely output variables to zero - for(int i = 0; i < mam4::aero_model::pcnst; ++i)Kokkos::deep_copy(ptend_q[i], 0.0); - Kokkos::deep_copy(factnum, 0.0); - Kokkos::deep_copy(tendnd, 0.0); - - //initialize all diagnostic ouputs to zero: - for(int i = 0; i < mam4::aero_model::pcnst; ++i){ - Kokkos::deep_copy(coltend[i], 0.0); - Kokkos::deep_copy(coltend_cw[i], 0.0); - } - Kokkos::deep_copy(qcld, 0.0); - Kokkos::deep_copy(ndropcol, 0.0); - Kokkos::deep_copy(ndropmix, 0.0); - Kokkos::deep_copy(nsource, 0.0); - Kokkos::deep_copy(wtke, 0.0); - Kokkos::deep_copy(ccn, 0.0); - - // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -949,6 +920,14 @@ void MAMAci::set_grids( add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); + // FIXME: [TEMPORARY]droplet number mixing ratio source tendency [#/kg/s] + add_field("nsource", scalar3d_layout_mid, n_unit / s, + grid_name); + + // FIXME: [TEMPORARY]droplet number mixing ratio tendency due to mixing [#/kg/s] + add_field("ndropmix", scalar3d_layout_mid, n_unit / s, + grid_name); + // ------------------------------------------------------------------------ // Output from hetrozenous freezing // ------------------------------------------------------------------------ @@ -1101,10 +1080,8 @@ void MAMAci::initialize_impl(const RunType run_type) { // hetrozenous freezing outputs hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); - hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); @@ -1165,10 +1142,15 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(ndropcol_, ncol_, nlev_); // droplet number mixing ratio tendency due to mixing [#/kg/s] - Kokkos::resize(ndropmix_, ncol_, nlev_); + //Kokkos::resize(ndropmix_, ncol_, nlev_); + //Temporarily output ndropmix_ + ndropmix_ = get_field_out("ndropmix").get_view(); // droplet number mixing ratio source tendency [#/kg/s] - Kokkos::resize(nsource_, ncol_, nlev_); + //Kokkos::resize(nsource_, ncol_, nlev_); + //Temporarily output nsource_ + nsource_ = get_field_out("nsource").get_view(); + // subgrid vertical velocity [m/s] Kokkos::resize(wtke_, ncol_, nlev_); From 3ce0a91a63ecc5cbb14a490e74f39bbceb612a20 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 14 May 2024 12:32:38 -0700 Subject: [PATCH 123/142] Adds a temporary varaible to capture nc before aci --- .../mam/eamxx_mam_aci_process_interface.cpp | 33 ++++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 1 + 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 5016c8bee82..2cf2d70fc8d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -921,11 +921,14 @@ void MAMAci::set_grids( grid_name); // FIXME: [TEMPORARY]droplet number mixing ratio source tendency [#/kg/s] - add_field("nsource", scalar3d_layout_mid, n_unit / s, - grid_name); + add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); + + // FIXME: [TEMPORARY]droplet number mixing ratio tendency due to mixing + // [#/kg/s] + add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); - // FIXME: [TEMPORARY]droplet number mixing ratio tendency due to mixing [#/kg/s] - add_field("ndropmix", scalar3d_layout_mid, n_unit / s, + // FIXME: [TEMPORARY]droplet number as seen by ACI [#/kg] + add_field("nc_inp_to_aci", scalar3d_layout_mid, n_unit / s, grid_name); // ------------------------------------------------------------------------ @@ -1142,15 +1145,17 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(ndropcol_, ncol_, nlev_); // droplet number mixing ratio tendency due to mixing [#/kg/s] - //Kokkos::resize(ndropmix_, ncol_, nlev_); - //Temporarily output ndropmix_ + // Kokkos::resize(ndropmix_, ncol_, nlev_); + // Temporarily output ndropmix_ ndropmix_ = get_field_out("ndropmix").get_view(); // droplet number mixing ratio source tendency [#/kg/s] - //Kokkos::resize(nsource_, ncol_, nlev_); - //Temporarily output nsource_ + // Kokkos::resize(nsource_, ncol_, nlev_); + // Temporarily output nsource_ nsource_ = get_field_out("nsource").get_view(); - + + // Temporarily output nc_inp_to_aci_ + nc_inp_to_aci_ = get_field_out("nc_inp_to_aci").get_view(); // subgrid vertical velocity [m/s] Kokkos::resize(wtke_, ncol_, nlev_); @@ -1238,6 +1243,16 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // FIXME: Temporary assignment of nc + // Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [=](int kk) { + nc_inp_to_aci_(icol, kk) = wet_atm_.nc(icol, kk); + }); + }); + compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 3f6f7a09314..f1a1f21d33f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -98,6 +98,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d ndropcol_; view_2d ndropmix_; view_2d nsource_; + view_2d nc_inp_to_aci_; // FIXME: TEMPORARY output view_2d wtke_; view_3d ccn_; view_2d coltend_[mam4::ndrop::ncnst_tot]; From 4f295d63c4cd48426fcfb36843f97bafcfeedf53 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 15 May 2024 17:18:44 -0700 Subject: [PATCH 124/142] Temporarily adds ccn diagnostics --- .../mam/eamxx_mam_aci_process_interface.cpp | 30 +++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 6 ++++ 2 files changed, 36 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 2cf2d70fc8d..036e4e903c2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -930,6 +930,15 @@ void MAMAci::set_grids( // FIXME: [TEMPORARY]droplet number as seen by ACI [#/kg] add_field("nc_inp_to_aci", scalar3d_layout_mid, n_unit / s, grid_name); + const auto cm_tmp = m / 100; // FIXME: [TEMPORARY] remove this + const auto cm3 = cm_tmp * cm_tmp * cm_tmp; // FIXME: [TEMPORARY] remove this + // FIXME: [TEMPORARY] remove the following ccn outputs + add_field("ccn_0p02", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p05", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p1", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p2", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p5", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_1p0", scalar3d_layout_mid, cm3, grid_name); // ------------------------------------------------------------------------ // Output from hetrozenous freezing @@ -1157,6 +1166,14 @@ void MAMAci::initialize_impl(const RunType run_type) { // Temporarily output nc_inp_to_aci_ nc_inp_to_aci_ = get_field_out("nc_inp_to_aci").get_view(); + // FIXME: [TEMPORARY] remove the following ccn outputs + ccn_0p02_ = get_field_out("ccn_0p02").get_view(); + ccn_0p05_ = get_field_out("ccn_0p05").get_view(); + ccn_0p1_ = get_field_out("ccn_0p1").get_view(); + ccn_0p2_ = get_field_out("ccn_0p2").get_view(); + ccn_0p5_ = get_field_out("ccn_0p5").get_view(); + ccn_1p0_ = get_field_out("ccn_1p0").get_view(); + // subgrid vertical velocity [m/s] Kokkos::resize(wtke_, ncol_, nlev_); @@ -1323,6 +1340,19 @@ void MAMAci::run_impl(const double dt) { dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. + Kokkos::deep_copy(ccn_0p02_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 0)); + Kokkos::deep_copy(ccn_0p05_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 1)); + Kokkos::deep_copy(ccn_0p1_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 2)); + Kokkos::deep_copy(ccn_0p2_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 3)); + Kokkos::deep_copy(ccn_0p5_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 4)); + Kokkos::deep_copy(ccn_1p0_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 5)); + //--------------------------------------------------------------------------- // NOTE: DO NOT UPDATE cloud borne aerosols using the qqcw_fld_work_ array // at this point as heterozenous freezing needs to use cloud borne aerosols diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index f1a1f21d33f..093dd571dd9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -99,6 +99,12 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d ndropmix_; view_2d nsource_; view_2d nc_inp_to_aci_; // FIXME: TEMPORARY output + view_2d ccn_0p02_; // FIXME: TEMPORARY output + view_2d ccn_0p05_; // FIXME: TEMPORARY output + view_2d ccn_0p1_; // FIXME: TEMPORARY output + view_2d ccn_0p2_; // FIXME: TEMPORARY output + view_2d ccn_0p5_; // FIXME: TEMPORARY output + view_2d ccn_1p0_; // FIXME: TEMPORARY output view_2d wtke_; view_3d ccn_; view_2d coltend_[mam4::ndrop::ncnst_tot]; From 380ad10e5c8eea1bb1a77b87592ab0bcbebc12bf Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 18 May 2024 21:28:33 -0700 Subject: [PATCH 125/142] Fixes optics test, this commit has ACI single grid cell printouts --- .../mam/eamxx_mam_aci_process_interface.cpp | 259 +++++++++++++++++- .../eamxx/src/physics/mam/mam_coupling.hpp | 16 +- .../single-process/mam/aci/CMakeLists.txt | 3 +- .../tests/single-process/mam/aci/input.yaml | 6 +- externals/mam4xx | 2 +- 5 files changed, 263 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 036e4e903c2..2916ee73725 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,4 +1,5 @@ #include +#include /* ----------------------------------------------------------------- @@ -21,6 +22,222 @@ namespace scream { namespace { +void print_output(const Real w0, const Real rho, const Real tke, + const Real wsub, const Real wice, const Real wsig, + const Real naai_hom, const Real naai, const Real rpdel, + MAMAci::view_3d factnum, const Real tendnd, + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + const Real hetfrz_immersion_nucleation_tend, + const Real hetfrz_contact_nucleation_tend, + const Real hetfrz_depostion_nucleation_tend, + const mam_coupling::AerosolState &dry_aero, const int kb) { + + std::cout << "w0:" << std::setprecision (15) << w0 << std::endl; + std::cout << "rho: " << rho << std::endl; + std::cout << "TKE:" << tke << std::endl; + std::cout << "wsub:" << wsub << std::endl; + std::cout << "wsubi:" << wice << std::endl; + std::cout << "wsig:" << wsig << std::endl; + std::cout << "dgnum_ait:"<< std::endl; + std::cout << "naai_:" << naai << std::endl; + std::cout << "naai_hom:" << naai_hom << std::endl; + //std::cout << "rpdel_:" << rpdel << std::endl; + std::cout << "factnum:" << factnum(0, 0, kb) << " " << factnum(0, 1, kb) + << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) + << std::endl; + std::cout << "nctend_mixnuc:" << tendnd << std::endl; + + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + for(int ic = 9; ic < 40; ++ic) { + std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; + } + /*for(int ic = 0; ic < 25; ++ic) { + std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) + << std::endl; + }*/ + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + + /*for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + }*/ + + std::cout << "hetfrz_immersion_nucleation_tend_:" + << hetfrz_immersion_nucleation_tend << ":" + << hetfrz_immersion_nucleation_tend << std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" + << hetfrz_contact_nucleation_tend << std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" + << hetfrz_depostion_nucleation_tend << std::endl; +} + +void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, + const int ncol_, const int nlev_) { + const Real w_sec_e3sm[73] = { + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.44201892319518146E-003, + 0.77315620137962326E-003, 0.24806301482800117E-002, + 0.11109772692898754E-001, 0.40651094032490273E-001, + 0.82156694426095800E-001, 0.12207124453993526E+000, + 0.15516728994634110E+000, 0.17775318086169636E+000, + 0.18549817250146838E+000, 0.17184548286554119E+000, + 0.12741230682196053E+000, 0.65495229516041628E-001, + 0.26909155217660592E-001}; + + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + w_sec_int_(icol, kk) = w_sec_e3sm[kk]; + // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + } + // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; + } + const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, + 0.75991761081225006E-002, 0.12291092068185365E-001, + 0.11484807652762415E-001, 0.10856880396302943E-001, + 0.10500384508819637E-001, 0.10361486171738229E-001, + 0.10333325067964508E-001, 0.10258838031435397E-001, + 0.10027325248446619E-001, 0.97784259072973521E-002, + 0.96611845055866539E-002, 0.96249746122327937E-002, + 0.95773431515696512E-002, 0.95180614513688099E-002, + 0.94713233348487150E-002, 0.94503864489758338E-002, + 0.94536294366578833E-002, 0.94575972194308883E-002, + 0.94403767489615684E-002, 0.93975694769176284E-002, + 0.93322843554751022E-002, 0.92777070192527501E-002, + 0.92456776697171228E-002, 0.92266924824142716E-002, + 0.92123025773060436E-002, 0.91888715633294191E-002, + 0.91516797753615851E-002, 0.90958299606649744E-002, + 0.89988037524983237E-002, 0.88220984587642423E-002, + 0.85231270833157156E-002, 0.81397522619395188E-002, + 0.79160421807845088E-002, 0.81206851117902653E-002, + 0.86526891616674779E-002, 0.91682975412125615E-002, + 0.96043394254592580E-002, 0.10033689085881327E-001, + 0.10428656694074272E-001, 0.10715913043864789E-001, + 0.10919631245454951E-001, 0.11250937075285789E-001, + 0.11829292157343831E-001, 0.12413311776454055E-001, + 0.12851317662157077E-001, 0.13175523677700330E-001, + 0.13224182907540188E-001, 0.13085937680733115E-001, + 0.12615055546741534E-001, 0.11995423733019836E-001, + 0.12346556881757400E-001, 0.13433752971524651E-001, + 0.13904308240950175E-001, 0.13539811748121957E-001, + 0.12555099320041433E-001, 0.11519643673351362E-001, + 0.11414071302852231E-001, 0.13409756835238139E-001, + 0.24071962815959351E-001, 0.75489419450816414E-001, + 0.62082011878960308E+000, 0.63952862312816796E+001, + 0.16226857944175123E+002, 0.21882852534279891E+002, + 0.24966173574402408E+002, 0.25710753126453692E+002, + 0.24069881024271943E+002, 0.19743922403487922E+002, + 0.98667814246712027E+001, 0.25633359450143991E+001, + 0.14682471685037493E+001}; + // compute eddy diffusivity of heat at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + // kvh_int_(icol, kk) = kvh_mid_(icol, kk); + kvh_int_(icol, kk) = kvh_e3sm[kk]; + } + // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; + } +} + +void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, + const int nlev_) { + const Real dgnum_ait_e3sm[72] = { + 0.20877713336487552E-007, 0.21782230353342090E-007, + 0.21688324003865861E-007, 0.21112855042342451E-007, + 0.19162058462939536E-007, 0.18102979880838476E-007, + 0.17906980715477606E-007, 0.20271254074583327E-007, + 0.22698983422181942E-007, 0.24134835117044986E-007, + 0.25498156808001372E-007, 0.29796738799905547E-007, + 0.35822987394021908E-007, 0.41170963764365215E-007, + 0.44892726528330642E-007, 0.47217231342203107E-007, + 0.48928661807108766E-007, 0.50170939816128735E-007, + 0.51078750853732200E-007, 0.52247333465736065E-007, + 0.53190758580174931E-007, 0.53576491941850044E-007, + 0.53915614473890715E-007, 0.54510964775236826E-007, + 0.55643231691556703E-007, 0.57057811112589899E-007, + 0.58177383586181116E-007, 0.58209849180850108E-007, + 0.57976751598840998E-007, 0.52000000000000002E-007, + 0.50728746567226150E-007, 0.49119902704480870E-007, + 0.48212162162050883E-007, 0.49227715213506454E-007, + 0.46876827233752246E-007, 0.45360603896257791E-007, + 0.49986783979004747E-007, 0.51186879246229022E-007, + 0.50009353247048599E-007, 0.48250264542204811E-007, + 0.47560278748093609E-007, 0.48298089720730957E-007, + 0.49095935613468768E-007, 0.49493024126912931E-007, + 0.50250797590476007E-007, 0.51949267668322422E-007, + 0.53778727208416418E-007, 0.53563593301099588E-007, + 0.51218136771199298E-007, 0.43171429694325200E-007, + 0.39019610039033895E-007, 0.36175109143257051E-007, + 0.42731638777892750E-007, 0.38060728507221777E-007, + 0.44046323901481340E-007, 0.39216732751330010E-007, + 0.34842233953609988E-007, 0.34068804733226066E-007, + 0.30636043694263528E-007, 0.28302341686131413E-007, + 0.33023014309036320E-007, 0.34745748365385196E-007, + 0.43623545003583371E-007, 0.48206451795644064E-007, + 0.49854490325455530E-007, 0.50346335647724146E-007, + 0.50661560988561763E-007, 0.50986261962838767E-007, + 0.51256955985111086E-007, 0.51482578449096488E-007, + 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + } + } +} + KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -490,7 +707,7 @@ void call_function_dropmixnuc( // get qqcw at a grid cell (col,lev) // NOTE: The layout for qqcw array is based on mam_idx in - // dropmixnuc To mimic that, we are using the following for-loops + // dropmixnuc. To mimic that, we are using the following for-loops int ind_qqcw = 0; for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { qqcw_view[ind_qqcw](klev) = @@ -834,7 +1051,8 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + //add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -848,8 +1066,8 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); - + //add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid = @@ -1054,12 +1272,14 @@ void MAMAci::initialize_impl(const RunType run_type) { wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + //std::cout<<"index:"<(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + //std::cout<<"index:"< 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); + //std::cout<<"indices:"< 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); + //std::cout<<"indices:"<::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // FIXME: Remove set_input and print_input + set_input(w_sec_int_, kvh_int_, ncol_, nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); // FIXME: Temporary assignment of nc // Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); @@ -1275,9 +1504,9 @@ void MAMAci::run_impl(const double dt) { w0_, rho_); // Get w_sec_int_ from w_sec_mid_ - compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + /*compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output - w_sec_int_); + w_sec_int_);*/ compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output @@ -1294,6 +1523,8 @@ void MAMAci::run_impl(const double dt) { compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); + // FIXME:Remove set_dgait + set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -1321,9 +1552,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. // Get kvh_int_ from kvh_mid_ - compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + /*compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, // output - kvh_int_); + kvh_int_);*/ // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial @@ -1385,6 +1616,14 @@ void MAMAci::run_impl(const double dt) { // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function + + const int kb = 62; + print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), + wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), + rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, + qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), + hetfrz_contact_nucleation_tend_(0, kb), + hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); } } // namespace scream diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index ae8c84210b5..ab7b18c83c7 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -548,10 +548,12 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), "int_aero_nmr not defined for dry aerosol state!"); progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); + //std::cout<<"progs.n_mode_i index:"< Date: Sat, 18 May 2024 21:40:56 -0700 Subject: [PATCH 126/142] Fixes optics tests --- .../mam/eamxx_mam_aci_process_interface.cpp | 257 +----------------- .../eamxx/src/physics/mam/mam_coupling.hpp | 18 +- .../single-process/mam/aci/CMakeLists.txt | 3 +- .../tests/single-process/mam/aci/input.yaml | 6 +- 4 files changed, 21 insertions(+), 263 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 2916ee73725..52da3ed0498 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,5 +1,4 @@ #include -#include /* ----------------------------------------------------------------- @@ -22,222 +21,6 @@ namespace scream { namespace { -void print_output(const Real w0, const Real rho, const Real tke, - const Real wsub, const Real wice, const Real wsig, - const Real naai_hom, const Real naai, const Real rpdel, - MAMAci::view_3d factnum, const Real tendnd, - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - const Real hetfrz_immersion_nucleation_tend, - const Real hetfrz_contact_nucleation_tend, - const Real hetfrz_depostion_nucleation_tend, - const mam_coupling::AerosolState &dry_aero, const int kb) { - - std::cout << "w0:" << std::setprecision (15) << w0 << std::endl; - std::cout << "rho: " << rho << std::endl; - std::cout << "TKE:" << tke << std::endl; - std::cout << "wsub:" << wsub << std::endl; - std::cout << "wsubi:" << wice << std::endl; - std::cout << "wsig:" << wsig << std::endl; - std::cout << "dgnum_ait:"<< std::endl; - std::cout << "naai_:" << naai << std::endl; - std::cout << "naai_hom:" << naai_hom << std::endl; - //std::cout << "rpdel_:" << rpdel << std::endl; - std::cout << "factnum:" << factnum(0, 0, kb) << " " << factnum(0, 1, kb) - << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) - << std::endl; - std::cout << "nctend_mixnuc:" << tendnd << std::endl; - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int ic = 9; ic < 40; ++ic) { - std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; - } - /*for(int ic = 0; ic < 25; ++ic) { - std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) - << std::endl; - }*/ - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - - /*for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - }*/ - - std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend << ":" - << hetfrz_immersion_nucleation_tend << std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" - << hetfrz_contact_nucleation_tend << std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" - << hetfrz_depostion_nucleation_tend << std::endl; -} - -void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, - const int ncol_, const int nlev_) { - const Real w_sec_e3sm[73] = { - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.44201892319518146E-003, - 0.77315620137962326E-003, 0.24806301482800117E-002, - 0.11109772692898754E-001, 0.40651094032490273E-001, - 0.82156694426095800E-001, 0.12207124453993526E+000, - 0.15516728994634110E+000, 0.17775318086169636E+000, - 0.18549817250146838E+000, 0.17184548286554119E+000, - 0.12741230682196053E+000, 0.65495229516041628E-001, - 0.26909155217660592E-001}; - - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_e3sm[kk]; - // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); - } - // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); - w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; - } - const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, - 0.75991761081225006E-002, 0.12291092068185365E-001, - 0.11484807652762415E-001, 0.10856880396302943E-001, - 0.10500384508819637E-001, 0.10361486171738229E-001, - 0.10333325067964508E-001, 0.10258838031435397E-001, - 0.10027325248446619E-001, 0.97784259072973521E-002, - 0.96611845055866539E-002, 0.96249746122327937E-002, - 0.95773431515696512E-002, 0.95180614513688099E-002, - 0.94713233348487150E-002, 0.94503864489758338E-002, - 0.94536294366578833E-002, 0.94575972194308883E-002, - 0.94403767489615684E-002, 0.93975694769176284E-002, - 0.93322843554751022E-002, 0.92777070192527501E-002, - 0.92456776697171228E-002, 0.92266924824142716E-002, - 0.92123025773060436E-002, 0.91888715633294191E-002, - 0.91516797753615851E-002, 0.90958299606649744E-002, - 0.89988037524983237E-002, 0.88220984587642423E-002, - 0.85231270833157156E-002, 0.81397522619395188E-002, - 0.79160421807845088E-002, 0.81206851117902653E-002, - 0.86526891616674779E-002, 0.91682975412125615E-002, - 0.96043394254592580E-002, 0.10033689085881327E-001, - 0.10428656694074272E-001, 0.10715913043864789E-001, - 0.10919631245454951E-001, 0.11250937075285789E-001, - 0.11829292157343831E-001, 0.12413311776454055E-001, - 0.12851317662157077E-001, 0.13175523677700330E-001, - 0.13224182907540188E-001, 0.13085937680733115E-001, - 0.12615055546741534E-001, 0.11995423733019836E-001, - 0.12346556881757400E-001, 0.13433752971524651E-001, - 0.13904308240950175E-001, 0.13539811748121957E-001, - 0.12555099320041433E-001, 0.11519643673351362E-001, - 0.11414071302852231E-001, 0.13409756835238139E-001, - 0.24071962815959351E-001, 0.75489419450816414E-001, - 0.62082011878960308E+000, 0.63952862312816796E+001, - 0.16226857944175123E+002, 0.21882852534279891E+002, - 0.24966173574402408E+002, 0.25710753126453692E+002, - 0.24069881024271943E+002, 0.19743922403487922E+002, - 0.98667814246712027E+001, 0.25633359450143991E+001, - 0.14682471685037493E+001}; - // compute eddy diffusivity of heat at the interfaces - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - // kvh_int_(icol, kk) = kvh_mid_(icol, kk); - kvh_int_(icol, kk) = kvh_e3sm[kk]; - } - // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); - kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; - } -} - -void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, - const int nlev_) { - const Real dgnum_ait_e3sm[72] = { - 0.20877713336487552E-007, 0.21782230353342090E-007, - 0.21688324003865861E-007, 0.21112855042342451E-007, - 0.19162058462939536E-007, 0.18102979880838476E-007, - 0.17906980715477606E-007, 0.20271254074583327E-007, - 0.22698983422181942E-007, 0.24134835117044986E-007, - 0.25498156808001372E-007, 0.29796738799905547E-007, - 0.35822987394021908E-007, 0.41170963764365215E-007, - 0.44892726528330642E-007, 0.47217231342203107E-007, - 0.48928661807108766E-007, 0.50170939816128735E-007, - 0.51078750853732200E-007, 0.52247333465736065E-007, - 0.53190758580174931E-007, 0.53576491941850044E-007, - 0.53915614473890715E-007, 0.54510964775236826E-007, - 0.55643231691556703E-007, 0.57057811112589899E-007, - 0.58177383586181116E-007, 0.58209849180850108E-007, - 0.57976751598840998E-007, 0.52000000000000002E-007, - 0.50728746567226150E-007, 0.49119902704480870E-007, - 0.48212162162050883E-007, 0.49227715213506454E-007, - 0.46876827233752246E-007, 0.45360603896257791E-007, - 0.49986783979004747E-007, 0.51186879246229022E-007, - 0.50009353247048599E-007, 0.48250264542204811E-007, - 0.47560278748093609E-007, 0.48298089720730957E-007, - 0.49095935613468768E-007, 0.49493024126912931E-007, - 0.50250797590476007E-007, 0.51949267668322422E-007, - 0.53778727208416418E-007, 0.53563593301099588E-007, - 0.51218136771199298E-007, 0.43171429694325200E-007, - 0.39019610039033895E-007, 0.36175109143257051E-007, - 0.42731638777892750E-007, 0.38060728507221777E-007, - 0.44046323901481340E-007, 0.39216732751330010E-007, - 0.34842233953609988E-007, 0.34068804733226066E-007, - 0.30636043694263528E-007, 0.28302341686131413E-007, - 0.33023014309036320E-007, 0.34745748365385196E-007, - 0.43623545003583371E-007, 0.48206451795644064E-007, - 0.49854490325455530E-007, 0.50346335647724146E-007, - 0.50661560988561763E-007, 0.50986261962838767E-007, - 0.51256955985111086E-007, 0.51482578449096488E-007, - 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - } - } -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -1051,8 +834,7 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - //add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); - add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -1066,8 +848,8 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - //add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid = @@ -1272,14 +1054,12 @@ void MAMAci::initialize_impl(const RunType run_type) { wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; - //std::cout<<"index:"<(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - //std::cout<<"index:"< 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - //std::cout<<"indices:"< 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); - //std::cout<<"indices:"<::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - - // FIXME: Remove set_input and print_input - set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); // FIXME: Temporary assignment of nc // Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); @@ -1504,9 +1275,9 @@ void MAMAci::run_impl(const double dt) { w0_, rho_); // Get w_sec_int_ from w_sec_mid_ - /*compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output - w_sec_int_);*/ + w_sec_int_); compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output @@ -1523,8 +1294,6 @@ void MAMAci::run_impl(const double dt) { compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - // FIXME:Remove set_dgait - set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -1552,9 +1321,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. // Get kvh_int_ from kvh_mid_ - /*compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, // output - kvh_int_);*/ + kvh_int_); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial @@ -1616,14 +1385,6 @@ void MAMAci::run_impl(const double dt) { // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function - - const int kb = 62; - print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), - wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), - rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, - qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), - hetfrz_contact_nucleation_tend_(0, kb), - hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); } } // namespace scream diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index ab7b18c83c7..2d89b573a54 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -548,12 +548,9 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), "int_aero_nmr not defined for dry aerosol state!"); progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); - //std::cout<<"progs.n_mode_i index:"< Date: Sat, 18 May 2024 21:48:21 -0700 Subject: [PATCH 127/142] Rebases mam4xx submodule --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 72219c8db1b..a8aa56ba915 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 72219c8db1b27ccacc3e906738b68dd639196961 +Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e From 423add534520db83abdbb807f01c0a1937438a65 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 24 May 2024 09:36:30 -0700 Subject: [PATCH 128/142] Moves aci functions to a separate hpp file --- .../physics/mam/eamxx_mam_aci_functions.hpp | 726 ++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.cpp | 717 +---------------- 2 files changed, 727 insertions(+), 716 deletions(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp new file mode 100644 index 00000000000..5174eb65236 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -0,0 +1,726 @@ +#ifndef EAMXX_MAM_ACI_FUNCTION_HPP +#define EAMXX_MAM_ACI_FUNCTION_HPP + +#include +#include +#include + +namespace scream { + +namespace { + +KOKKOS_INLINE_FUNCTION +void compute_w0_and_rho(const haero::ThreadTeam &team, + const MAMAci::const_view_2d omega, + const MAMAci::const_view_2d T_mid, + const MAMAci::const_view_2d p_mid, const int icol, + const int top_lev, const int nlev, + // output + MAMAci::view_2d w0, MAMAci::view_2d rho) { + // Get physical constants + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + // Gas constant for dry air [J/(kg*K) or J/Kg/K] + static constexpr auto rair = C::Rair; + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol, kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); + w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); + }); +} +void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const int top_lev, const int nlev, + // output + MAMAci::view_2d w0, MAMAci::view_2d rho) { + MAMAci::const_view_2d omega = dry_atmosphere.omega; + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_w0_and_rho(team, omega, T_mid, p_mid, icol, top_lev, nlev, + // output + w0, rho); + }); +} + +void compute_values_at_interfaces(haero::ThreadTeamPolicy team_policy, + const MAMAci::const_view_2d var_mid, + const MAMAci::view_2d dz, const int nlev_, + // output + MAMAci::view_2d var_int) { + using CO = scream::ColumnOps; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + const auto var_mid_col = ekat::subview(var_mid, icol); + const auto var_int_col = ekat::subview(var_int, icol); + const auto dz_col = ekat::subview(dz, icol); + + const Real bc_top = var_mid_col(0); + const Real bc_bot = var_mid_col(nlev_ - 1); + + CO::compute_interface_values_linear(team, nlev_, var_mid_col, dz_col, + bc_top, bc_bot, var_int_col); + }); +} + +KOKKOS_INLINE_FUNCTION +void compute_tke_using_w_sec(const haero::ThreadTeam &team, + const MAMAci::const_view_2d w_sec, const int icol, + const int nlev, + // output + MAMAci::view_2d tke) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev + 1), + KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); +} +void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, + const MAMAci::const_view_2d w_sec, const int nlev, + // output + MAMAci::view_2d tke) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_tke_using_w_sec(team, w_sec, icol, nlev, + // output + tke); + }); +} +KOKKOS_INLINE_FUNCTION +void compute_subgrid_scale_velocities( + const haero::ThreadTeam &team, const MAMAci::const_view_2d tke, + const Real wsubmin, const int icol, const int top_lev, const int nlev, + // output + MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol, kk) = wsubmin; + wsubice(icol, kk) = 0.001; + wsig(icol, kk) = 0.001; + }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * + (2.0 / 3.0)); + wsig(icol, kk) = + mam4::utils::min_max_bound(0.001, 10.0, wsub(icol, kk)); + wsubice(icol, kk) = + mam4::utils::min_max_bound(0.2, 10.0, wsub(icol, kk)); + wsub(icol, kk) = haero::max(wsubmin, wsub(icol, kk)); + }); +} +void compute_subgrid_scale_velocities( + haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d tke, + const Real wsubmin, const int top_lev, const int nlev, + // output + MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_subgrid_scale_velocities(team, tke, wsubmin, icol, top_lev, + nlev, + // output + wsub, wsubice, wsig); + }); +} + +KOKKOS_INLINE_FUNCTION +void compute_aitken_dry_diameter(const haero::ThreadTeam &team, + const MAMAci::const_view_3d dgnum, + const int icol, const int top_lev, + const int nlev, + // output + MAMAci::view_2d aitken_dry_dia) { + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); + }); +} +void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, + const MAMAci::const_view_3d dgnum, + const int top_lev, const int nlev, + // output + MAMAci::view_2d aitken_dry_dia) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_aitken_dry_diameter(team, dgnum, icol, top_lev, nlev, + // output + aitken_dry_dia); + }); +} + +void compute_nucleate_ice_tendencies( + const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const mam_coupling::AerosolState &dry_aero, const MAMAci::view_2d wsubice, + const MAMAci::view_2d aitken_dry_dia, const int nlev, const double dt, + // output + MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, + MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, + // ## output used by other processes ## + MAMAci::view_2d naai) { + //------------------------------------------------------------- + // Get number of activated aerosol for ice nucleation (naai) + // from ice nucleation + //------------------------------------------------------------- + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + //--------------------------------------------------------------------- + // Set up surface, pronostics atmosphere, diagnostics, and tendencies + // classes. + //--------------------------------------------------------------------- + + // For compute_tendecies interface only, this structure is empty + haero::Surface surf{}; + + // Store interstitial and cld borne aerosols in "progrs" struture + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aero, icol); + + // Store atmopsheric vars (Tmid, Pmid, cloud fraction, qv, wsubmin) + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + + // Update the updraft velocity needed by nucleation to be "wsubice" + // in the haero_atm object + haero_atm.updraft_vel_ice_nucleation = ekat::subview(wsubice, icol); + + // All the output from this process is diagnotics; creates "diags" with + // nlev column length + mam4::Diagnostics diags(nlev); + + // Aitken mode index + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + diags.dry_geometric_mean_diameter_i[aitken_idx] = + ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated + // in nucleate_ice.compute_tendencies. + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); // not used + const mam4::AeroConfig aero_config; + const Real t = 0; // not used + nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, + surf, progs, diags, tends); + }); +} +KOKKOS_INLINE_FUNCTION +void store_liquid_cloud_fraction( + const haero::ThreadTeam &team, const MAMAci::const_view_2d qc, + const MAMAci::const_view_2d qi, const MAMAci::const_view_2d liqcldf, + const MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, + const int nlev, + // output + MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { + //------------------------------------------------------------- + // Get old and new liquid cloud fractions when amount of cloud + // is above qsmall threshold value + //------------------------------------------------------------- + // cut-off for cloud amount (ice or liquid) + static constexpr auto qsmall = 1e-18; + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + if((qc(icol, kk) + qi(icol, kk)) > qsmall) { + cloud_frac(icol, kk) = liqcldf(icol, kk); + cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); + } else { + cloud_frac(icol, kk) = 0; + cloud_frac_prev(icol, kk) = 0; + } + }); +} +void store_liquid_cloud_fraction( + haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const MAMAci::const_view_2d liqcldf, + const MAMAci::const_view_2d liqcldf_prev, const int top_lev, const int nlev, + // output + MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { + MAMAci::const_view_2d qc = dry_atmosphere.qc; + MAMAci::const_view_2d qi = dry_atmosphere.qi; + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + store_liquid_cloud_fraction(team, qc, qi, liqcldf, liqcldf_prev, icol, + top_lev, nlev, + // output + cloud_frac, cloud_frac_prev); + }); +} +KOKKOS_INLINE_FUNCTION +void compute_recipical_pseudo_density(const haero::ThreadTeam &team, + const MAMAci::const_view_2d pdel, + const int icol, const int nlev, + // output + MAMAci::view_2d rpdel) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); +} +void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, + MAMAci::const_view_2d pdel, + const int nlev, + // output + MAMAci::view_2d rpdel) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_recipical_pseudo_density(team, pdel, icol, nlev, + // output + rpdel); + }); +} + +void call_function_dropmixnuc( + haero::ThreadTeamPolicy team_policy, const Real dt, + mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, + const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, + const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, + const mam_coupling::AerosolState &dry_aero, const int nlev, + + // Following outputs are all diagnostics + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, + MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, + MAMAci::view_2d wtke, MAMAci::view_3d ccn, + + // ## outputs to be used by other processes ## + // qqcw_fld_work should be directly assigned to the cloud borne aerosols + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + + // ptend_q are the tendencies to the interstitial aerosols + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + + // factnum is used by the hetrozenous freezing + MAMAci::view_3d factnum, + + // tendnd is used by microphysics scheme (e.g. P3) + MAMAci::view_2d tendnd, + + // ## work arrays ## + MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], + MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, + MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { + // Extract atmosphere variables + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + MAMAci::const_view_2d zm = dry_atmosphere.z_mid; + MAMAci::const_view_2d pdel = dry_atmosphere.p_del; + MAMAci::const_view_2d p_int = dry_atmosphere.p_int; + MAMAci::const_view_2d nc = dry_atmosphere.nc; + + //---------------------------------------------------------------------- + // ## Declare local variables for class variables + //(FIXME: GPU hack, revisit this) + //---------------------------------------------------------------------- + MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; + MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; + MAMAci::view_2d loc_qqcw[mam4::ndrop::ncnst_tot]; + MAMAci::view_2d loc_ptend_q[mam4::aero_model::pcnst]; + MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; + MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; + + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + loc_raercol_cw[i][j] = raercol_cw[i][j]; + loc_raercol[i][j] = raercol[i][j]; + } + } + + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + loc_coltend[i] = coltend[i]; + loc_coltend_cw[i] = coltend_cw[i]; + } + + for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; + + MAMAci::view_2d qqcw_fld_work_loc[25]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + qqcw_fld_work_loc[i] = qqcw_fld_work[i]; + + MAMAci::view_3d state_q_work_loc = state_q_work; + + //---------------------------------------------------------------------- + // ## Assign scratch memory for work variables + //---------------------------------------------------------------------- + + MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; + MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; + MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; + MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; + MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; + MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; + MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; + MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; + MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; + MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; + MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; + MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; + MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; + MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; + MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; + + //--------------------------------------------------------------------------- + // ## Initialize the ndrop class. + //--------------------------------------------------------------------------- + const int ntot_amode = mam_coupling::num_aero_modes(); + const int maxd_aspectype = mam4::ndrop::maxd_aspectype; + const int nspec_max = mam4::ndrop::nspec_max; + int nspec_amode[ntot_amode] = {}; + int lspectype_amode[maxd_aspectype][ntot_amode] = {}; + int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; + int numptr_amode[ntot_amode] = {}; + int mam_idx[ntot_amode][nspec_max] = {}; + int mam_cnst_idx[ntot_amode][nspec_max] = {}; + + Real specdens_amode[maxd_aspectype] = {}; + Real spechygro[maxd_aspectype] = {}; + Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, + num2vol_ratio_min_nmodes[ntot_amode] = {}, + num2vol_ratio_max_nmodes[ntot_amode] = {}; + Real aten = 0; + mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, + numptr_amode, specdens_amode, spechygro, + mam_idx, mam_cnst_idx); + mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, + num2vol_ratio_max_nmodes); + //--------------------------------------------------------------------------- + //--------------------------------------------------------------------------- + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // for (int icol=0; icol<5; ++icol){ + mam4::ndrop::View1D raercol_cw_view[mam4::ndrop::pver][2]; + mam4::ndrop::View1D raercol_view[mam4::ndrop::pver][2]; + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + raercol_cw_view[i][j] = ekat::subview(loc_raercol_cw[i][j], icol); + raercol_view[i][j] = ekat::subview(loc_raercol[i][j], icol); + } + } + mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); + } + mam4::ColumnView ptend_q_view[mam4::aero_model::pcnst]; + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { + ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); + } + mam4::ColumnView coltend_view[mam4::ndrop::ncnst_tot], + coltend_cw_view[mam4::ndrop::ncnst_tot]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + coltend_view[i] = ekat::subview(loc_coltend[i], icol); + coltend_cw_view[i] = ekat::subview(loc_coltend_cw[i], icol); + } + + // To construct state_q, we need fields from Prognostics (aerosols) + // and Atmosphere (water species such as qv, qc etc.) + + // get prognostics + mam4::Prognostics progs_at_col = aerosols_for_column(dry_aero, icol); + + // get atmospheric quantities + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + + // Construct state_q (interstitial) and qqcw (cloud borne) arrays + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + [=](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; + ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } + + // get qqcw at a grid cell (col,lev) + // NOTE: The layout for qqcw array is based on mam_idx in + // dropmixnuc. To mimic that, we are using the following for-loops + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + qqcw_view[ind_qqcw](klev) = + dry_aero.cld_aero_nmr[m](icol, klev); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + qqcw_view[ind_qqcw](klev) = + dry_aero.cld_aero_mmr[m][a](icol, klev); + ++ind_qqcw; + } + } + } + }); + + mam4::ndrop::dropmixnuc( + team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), + ekat::subview(p_int, icol), ekat::subview(pdel, icol), + ekat::subview(rpdel, icol), + // in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] + ekat::subview(zm, icol), ekat::subview(state_q_work_loc, icol), + ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] + ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, + spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, + num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, + alogsig, aten, mam_idx, mam_cnst_idx, + ekat::subview(qcld, icol), // out + ekat::subview(wsub, icol), // in + ekat::subview(cloud_frac_prev, icol), // in + qqcw_view, // inout + ptend_q_view, ekat::subview(tendnd, icol), + ekat::subview(factnum, icol), ekat::subview(ndropcol, icol), + ekat::subview(ndropmix, icol), ekat::subview(nsource, icol), + ekat::subview(wtke, icol), ekat::subview(ccn, icol), coltend_view, + coltend_cw_view, + // work arrays + raercol_cw_view, raercol_view, ekat::subview(nact, icol), + ekat::subview(mact, icol), ekat::subview(eddy_diff, icol), + ekat::subview(zn, icol), ekat::subview(csbot, icol), + ekat::subview(zs, icol), ekat::subview(overlapp, icol), + ekat::subview(overlapm, icol), ekat::subview(eddy_diff_kp, icol), + ekat::subview(eddy_diff_km, icol), ekat::subview(qncld, icol), + ekat::subview(srcn, icol), ekat::subview(source, icol), + ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), + ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)); + }); +} + +// Update cloud borne aerosols +void update_cloud_borne_aerosols( + haero::ThreadTeamPolicy team_policy, + const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, + // output + mam_coupling::AerosolState &dry_aero) { + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], qqcw_fld_work[ind_qqcw]); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], qqcw_fld_work[ind_qqcw]); + ++ind_qqcw; + } + } + } +} + +// Update interstitial aerosols using tendencies - levels +void update_interstitial_aerosols_levs( + const haero::ThreadTeam &team, const int nlev, const int icol, + const int s_idx, const Real dt, + const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + // output + MAMAci::view_2d aero_mr) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [=](int kk) { + aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; + }); +} + +// Update interstitial aerosols using tendencies- cols and levs +void update_interstitial_aerosols( + haero::ThreadTeamPolicy team_policy, + const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], const int nlev, + const Real dt, + // output + mam_coupling::AerosolState &dry_aero) { + // starting index of ptend_q array (for MAM4, pcnst=40, ncnst_tot=25 ) + int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // loop through all modes and species + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + // mass mixing ratio of species "a" of mode "m" + auto aero_mmr = dry_aero.int_aero_mmr[m][a]; + + if(aero_mmr.data()) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // update values for all levs at this column + update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, + ptend_q, + // output + aero_mmr); + }); + // update index for the next species (only if aero_mmr.data() is True) + ++s_idx; + } + } + auto aero_nmr = + dry_aero.int_aero_nmr[m]; // number mixing ratio for mode "m" + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // update values for all levs at this column + update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, + ptend_q, + // output + aero_nmr); + }); + ++s_idx; // update index for the next species + } +} + +void call_hetfrz_compute_tendencies( + haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, + mam_coupling::DryAtmosphere &dry_atm_, + mam_coupling::AerosolState &dry_aero_, MAMAci::view_3d factnum, + const double dt, const int nlev, + // output + MAMAci::view_2d hetfrz_immersion_nucleation_tend, + MAMAci::view_2d hetfrz_contact_nucleation_tend, + MAMAci::view_2d hetfrz_depostion_nucleation_tend, + MAMAci::view_2d diagnostic_scratch_[]) { + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState dry_aero = dry_aero_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; + + MAMAci::view_2d diagnostic_scratch[MAMAci::hetro_scratch_]; + for(int i = 0; i < MAMAci::hetro_scratch_; ++i) + diagnostic_scratch[i] = diagnostic_scratch_[i]; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // Set up an atmosphere, surface, diagnostics, pronostics and + // tendencies class. + + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aero, icol); + + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + + mam4::Diagnostics diags(nlev); + + diags.activation_fraction[accum_idx] = + ekat::subview(factnum, icol, accum_idx); + + diags.activation_fraction[coarse_idx] = + ekat::subview(factnum, icol, coarse_idx); + + // These are the output tendencies from heterogeneous freezing that need + // to be added correctly to the cloud-micorphysics scheme. + diags.hetfrz_immersion_nucleation_tend = + ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = + ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = + ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + diags.stratiform_cloud_fraction = + ekat::subview(diagnostic_scratch[42], icol); + + // assign cloud fraction + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + [=](int klev) { + diags.stratiform_cloud_fraction(klev) = + haero_atm.cloud_fraction(klev); + }); + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0; // not used + hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, + progs, diags, tends); + }); +} +} // namespace +} //namespace scream + +#endif // ifdef EAMXX_MAM_ACI_FUNCTION_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 52da3ed0498..67791fe09af 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,4 +1,5 @@ #include +#include /* ----------------------------------------------------------------- @@ -18,722 +19,6 @@ FUTURE WORK: */ namespace scream { - -namespace { - -KOKKOS_INLINE_FUNCTION -void compute_w0_and_rho(const haero::ThreadTeam &team, - const MAMAci::const_view_2d omega, - const MAMAci::const_view_2d T_mid, - const MAMAci::const_view_2d p_mid, const int icol, - const int top_lev, const int nlev, - // output - MAMAci::view_2d w0, MAMAci::view_2d rho) { - // Get physical constants - using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - // Gas constant for dry air [J/(kg*K) or J/Kg/K] - static constexpr auto rair = C::Rair; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - w0(icol, kk) = 0; - rho(icol, kk) = -999.0; - }); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); - w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); - }); -} -void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, - const mam_coupling::DryAtmosphere &dry_atmosphere, - const int top_lev, const int nlev, - // output - MAMAci::view_2d w0, MAMAci::view_2d rho) { - MAMAci::const_view_2d omega = dry_atmosphere.omega; - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_w0_and_rho(team, omega, T_mid, p_mid, icol, top_lev, nlev, - // output - w0, rho); - }); -} - -void compute_values_at_interfaces(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_2d var_mid, - const MAMAci::view_2d dz, const int nlev_, - // output - MAMAci::view_2d var_int) { - using CO = scream::ColumnOps; - - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - const auto var_mid_col = ekat::subview(var_mid, icol); - const auto var_int_col = ekat::subview(var_int, icol); - const auto dz_col = ekat::subview(dz, icol); - - const Real bc_top = var_mid_col(0); - const Real bc_bot = var_mid_col(nlev_ - 1); - - CO::compute_interface_values_linear(team, nlev_, var_mid_col, dz_col, - bc_top, bc_bot, var_int_col); - }); -} - -KOKKOS_INLINE_FUNCTION -void compute_tke_using_w_sec(const haero::ThreadTeam &team, - const MAMAci::const_view_2d w_sec, const int icol, - const int nlev, - // output - MAMAci::view_2d tke) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev + 1), - KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); -} -void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_2d w_sec, const int nlev, - // output - MAMAci::view_2d tke) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_tke_using_w_sec(team, w_sec, icol, nlev, - // output - tke); - }); -} -KOKKOS_INLINE_FUNCTION -void compute_subgrid_scale_velocities( - const haero::ThreadTeam &team, const MAMAci::const_view_2d tke, - const Real wsubmin, const int icol, const int top_lev, const int nlev, - // output - MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { - // More refined computation of sub-grid vertical velocity - // Set to be zero at the surface by initialization. - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - wsub(icol, kk) = wsubmin; - wsubice(icol, kk) = 0.001; - wsig(icol, kk) = 0.001; - }); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * - (2.0 / 3.0)); - wsig(icol, kk) = - mam4::utils::min_max_bound(0.001, 10.0, wsub(icol, kk)); - wsubice(icol, kk) = - mam4::utils::min_max_bound(0.2, 10.0, wsub(icol, kk)); - wsub(icol, kk) = haero::max(wsubmin, wsub(icol, kk)); - }); -} -void compute_subgrid_scale_velocities( - haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d tke, - const Real wsubmin, const int top_lev, const int nlev, - // output - MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_subgrid_scale_velocities(team, tke, wsubmin, icol, top_lev, - nlev, - // output - wsub, wsubice, wsig); - }); -} - -KOKKOS_INLINE_FUNCTION -void compute_aitken_dry_diameter(const haero::ThreadTeam &team, - const MAMAci::const_view_3d dgnum, - const int icol, const int top_lev, - const int nlev, - // output - MAMAci::view_2d aitken_dry_dia) { - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); - }); -} -void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_3d dgnum, - const int top_lev, const int nlev, - // output - MAMAci::view_2d aitken_dry_dia) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_aitken_dry_diameter(team, dgnum, icol, top_lev, nlev, - // output - aitken_dry_dia); - }); -} - -void compute_nucleate_ice_tendencies( - const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, - const mam_coupling::DryAtmosphere &dry_atmosphere, - const mam_coupling::AerosolState &dry_aero, const MAMAci::view_2d wsubice, - const MAMAci::view_2d aitken_dry_dia, const int nlev, const double dt, - // output - MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, - MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, - // ## output used by other processes ## - MAMAci::view_2d naai) { - //------------------------------------------------------------- - // Get number of activated aerosol for ice nucleation (naai) - // from ice nucleation - //------------------------------------------------------------- - - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - //--------------------------------------------------------------------- - // Set up surface, pronostics atmosphere, diagnostics, and tendencies - // classes. - //--------------------------------------------------------------------- - - // For compute_tendecies interface only, this structure is empty - haero::Surface surf{}; - - // Store interstitial and cld borne aerosols in "progrs" struture - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aero, icol); - - // Store atmopsheric vars (Tmid, Pmid, cloud fraction, qv, wsubmin) - haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atmosphere, icol); - - // Update the updraft velocity needed by nucleation to be "wsubice" - // in the haero_atm object - haero_atm.updraft_vel_ice_nucleation = ekat::subview(wsubice, icol); - - // All the output from this process is diagnotics; creates "diags" with - // nlev column length - mam4::Diagnostics diags(nlev); - - // Aitken mode index - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - diags.dry_geometric_mean_diameter_i[aitken_idx] = - ekat::subview(aitken_dry_dia, icol); - - // These are the fields that are updated. Taking subviews means that - // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated - // in nucleate_ice.compute_tendencies. - diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); - diags.icenuc_num_immfrz = ekat::subview(niim, icol); - diags.icenuc_num_depnuc = ekat::subview(nidep, icol); - diags.icenuc_num_meydep = ekat::subview(nimey, icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); // not used - const mam4::AeroConfig aero_config; - const Real t = 0; // not used - nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, - surf, progs, diags, tends); - }); -} -KOKKOS_INLINE_FUNCTION -void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, const MAMAci::const_view_2d qc, - const MAMAci::const_view_2d qi, const MAMAci::const_view_2d liqcldf, - const MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, - const int nlev, - // output - MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { - //------------------------------------------------------------- - // Get old and new liquid cloud fractions when amount of cloud - // is above qsmall threshold value - //------------------------------------------------------------- - // cut-off for cloud amount (ice or liquid) - static constexpr auto qsmall = 1e-18; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - if((qc(icol, kk) + qi(icol, kk)) > qsmall) { - cloud_frac(icol, kk) = liqcldf(icol, kk); - cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); - } else { - cloud_frac(icol, kk) = 0; - cloud_frac_prev(icol, kk) = 0; - } - }); -} -void store_liquid_cloud_fraction( - haero::ThreadTeamPolicy team_policy, - const mam_coupling::DryAtmosphere &dry_atmosphere, - const MAMAci::const_view_2d liqcldf, - const MAMAci::const_view_2d liqcldf_prev, const int top_lev, const int nlev, - // output - MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { - MAMAci::const_view_2d qc = dry_atmosphere.qc; - MAMAci::const_view_2d qi = dry_atmosphere.qi; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, qc, qi, liqcldf, liqcldf_prev, icol, - top_lev, nlev, - // output - cloud_frac, cloud_frac_prev); - }); -} -KOKKOS_INLINE_FUNCTION -void compute_recipical_pseudo_density(const haero::ThreadTeam &team, - const MAMAci::const_view_2d pdel, - const int icol, const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), - "Error: pdel should be > 0.\n"); - rpdel(icol, kk) = 1 / pdel(icol, kk); - }); -} -void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, - MAMAci::const_view_2d pdel, - const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_recipical_pseudo_density(team, pdel, icol, nlev, - // output - rpdel); - }); -} - -void call_function_dropmixnuc( - haero::ThreadTeamPolicy team_policy, const Real dt, - mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, - const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, - const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, - const mam_coupling::AerosolState &dry_aero, const int nlev, - - // Following outputs are all diagnostics - MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, - MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, - MAMAci::view_2d wtke, MAMAci::view_3d ccn, - - // ## outputs to be used by other processes ## - // qqcw_fld_work should be directly assigned to the cloud borne aerosols - MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - - // ptend_q are the tendencies to the interstitial aerosols - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - - // factnum is used by the hetrozenous freezing - MAMAci::view_3d factnum, - - // tendnd is used by microphysics scheme (e.g. P3) - MAMAci::view_2d tendnd, - - // ## work arrays ## - MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], - MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, - MAMAci::view_3d nact, MAMAci::view_3d mact, - MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { - // Extract atmosphere variables - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - MAMAci::const_view_2d zm = dry_atmosphere.z_mid; - MAMAci::const_view_2d pdel = dry_atmosphere.p_del; - MAMAci::const_view_2d p_int = dry_atmosphere.p_int; - MAMAci::const_view_2d nc = dry_atmosphere.nc; - - //---------------------------------------------------------------------- - // ## Declare local variables for class variables - //(FIXME: GPU hack, revisit this) - //---------------------------------------------------------------------- - MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; - MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; - MAMAci::view_2d loc_qqcw[mam4::ndrop::ncnst_tot]; - MAMAci::view_2d loc_ptend_q[mam4::aero_model::pcnst]; - MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; - MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; - - for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { - loc_raercol_cw[i][j] = raercol_cw[i][j]; - loc_raercol[i][j] = raercol[i][j]; - } - } - - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - loc_coltend[i] = coltend[i]; - loc_coltend_cw[i] = coltend_cw[i]; - } - - for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; - - MAMAci::view_2d qqcw_fld_work_loc[25]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) - qqcw_fld_work_loc[i] = qqcw_fld_work[i]; - - MAMAci::view_3d state_q_work_loc = state_q_work; - - //---------------------------------------------------------------------- - // ## Assign scratch memory for work variables - //---------------------------------------------------------------------- - - MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; - MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; - MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; - MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; - MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; - MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; - MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; - MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; - MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; - MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; - MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; - MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; - MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; - MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; - MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; - - //--------------------------------------------------------------------------- - // ## Initialize the ndrop class. - //--------------------------------------------------------------------------- - const int ntot_amode = mam_coupling::num_aero_modes(); - const int maxd_aspectype = mam4::ndrop::maxd_aspectype; - const int nspec_max = mam4::ndrop::nspec_max; - int nspec_amode[ntot_amode] = {}; - int lspectype_amode[maxd_aspectype][ntot_amode] = {}; - int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; - int numptr_amode[ntot_amode] = {}; - int mam_idx[ntot_amode][nspec_max] = {}; - int mam_cnst_idx[ntot_amode][nspec_max] = {}; - - Real specdens_amode[maxd_aspectype] = {}; - Real spechygro[maxd_aspectype] = {}; - Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, - num2vol_ratio_min_nmodes[ntot_amode] = {}, - num2vol_ratio_max_nmodes[ntot_amode] = {}; - Real aten = 0; - mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, - numptr_amode, specdens_amode, spechygro, - mam_idx, mam_cnst_idx); - mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, - num2vol_ratio_max_nmodes); - //--------------------------------------------------------------------------- - //--------------------------------------------------------------------------- - - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - // for (int icol=0; icol<5; ++icol){ - mam4::ndrop::View1D raercol_cw_view[mam4::ndrop::pver][2]; - mam4::ndrop::View1D raercol_view[mam4::ndrop::pver][2]; - for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { - raercol_cw_view[i][j] = ekat::subview(loc_raercol_cw[i][j], icol); - raercol_view[i][j] = ekat::subview(loc_raercol[i][j], icol); - } - } - mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); - } - mam4::ColumnView ptend_q_view[mam4::aero_model::pcnst]; - for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); - } - mam4::ColumnView coltend_view[mam4::ndrop::ncnst_tot], - coltend_cw_view[mam4::ndrop::ncnst_tot]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - coltend_view[i] = ekat::subview(loc_coltend[i], icol); - coltend_cw_view[i] = ekat::subview(loc_coltend_cw[i], icol); - } - - // To construct state_q, we need fields from Prognostics (aerosols) - // and Atmosphere (water species such as qv, qc etc.) - - // get prognostics - mam4::Prognostics progs_at_col = aerosols_for_column(dry_aero, icol); - - // get atmospheric quantities - haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atmosphere, icol); - - // Construct state_q (interstitial) and qqcw (cloud borne) arrays - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - [=](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_lev_col, klev); - - // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // create colum views of state_q - for(int icnst = istart; icnst < mam4::aero_model::pcnst; - ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - } - - // get qqcw at a grid cell (col,lev) - // NOTE: The layout for qqcw array is based on mam_idx in - // dropmixnuc. To mimic that, we are using the following for-loops - int ind_qqcw = 0; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - qqcw_view[ind_qqcw](klev) = - dry_aero.cld_aero_nmr[m](icol, klev); - ++ind_qqcw; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - qqcw_view[ind_qqcw](klev) = - dry_aero.cld_aero_mmr[m][a](icol, klev); - ++ind_qqcw; - } - } - } - }); - - mam4::ndrop::dropmixnuc( - team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), - ekat::subview(p_int, icol), ekat::subview(pdel, icol), - ekat::subview(rpdel, icol), - // in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - ekat::subview(zm, icol), ekat::subview(state_q_work_loc, icol), - ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] - ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, - spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, - num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, - alogsig, aten, mam_idx, mam_cnst_idx, - ekat::subview(qcld, icol), // out - ekat::subview(wsub, icol), // in - ekat::subview(cloud_frac_prev, icol), // in - qqcw_view, // inout - ptend_q_view, ekat::subview(tendnd, icol), - ekat::subview(factnum, icol), ekat::subview(ndropcol, icol), - ekat::subview(ndropmix, icol), ekat::subview(nsource, icol), - ekat::subview(wtke, icol), ekat::subview(ccn, icol), coltend_view, - coltend_cw_view, - // work arrays - raercol_cw_view, raercol_view, ekat::subview(nact, icol), - ekat::subview(mact, icol), ekat::subview(eddy_diff, icol), - ekat::subview(zn, icol), ekat::subview(csbot, icol), - ekat::subview(zs, icol), ekat::subview(overlapp, icol), - ekat::subview(overlapm, icol), ekat::subview(eddy_diff_kp, icol), - ekat::subview(eddy_diff_km, icol), ekat::subview(qncld, icol), - ekat::subview(srcn, icol), ekat::subview(source, icol), - ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), - ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)); - }); -} - -// Update cloud borne aerosols -void update_cloud_borne_aerosols( - haero::ThreadTeamPolicy team_policy, - const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, - // output - mam_coupling::AerosolState &dry_aero) { - int ind_qqcw = 0; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], qqcw_fld_work[ind_qqcw]); - ++ind_qqcw; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], qqcw_fld_work[ind_qqcw]); - ++ind_qqcw; - } - } - } -} - -// Update interstitial aerosols using tendencies - levels -void update_interstitial_aerosols_levs( - const haero::ThreadTeam &team, const int nlev, const int icol, - const int s_idx, const Real dt, - const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - // output - MAMAci::view_2d aero_mr) { - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [=](int kk) { - aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; - }); -} - -// Update interstitial aerosols using tendencies- cols and levs -void update_interstitial_aerosols( - haero::ThreadTeamPolicy team_policy, - const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], const int nlev, - const Real dt, - // output - mam_coupling::AerosolState &dry_aero) { - // starting index of ptend_q array (for MAM4, pcnst=40, ncnst_tot=25 ) - int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // loop through all modes and species - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - for(int a = 0; a < mam4::num_species_mode(m); ++a) { - // mass mixing ratio of species "a" of mode "m" - auto aero_mmr = dry_aero.int_aero_mmr[m][a]; - - if(aero_mmr.data()) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, - ptend_q, - // output - aero_mmr); - }); - // update index for the next species (only if aero_mmr.data() is True) - ++s_idx; - } - } - auto aero_nmr = - dry_aero.int_aero_nmr[m]; // number mixing ratio for mode "m" - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, - ptend_q, - // output - aero_nmr); - }); - ++s_idx; // update index for the next species - } -} - -void call_hetfrz_compute_tendencies( - haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, - mam_coupling::DryAtmosphere &dry_atm_, - mam_coupling::AerosolState &dry_aero_, MAMAci::view_3d factnum, - const double dt, const int nlev, - // output - MAMAci::view_2d hetfrz_immersion_nucleation_tend, - MAMAci::view_2d hetfrz_contact_nucleation_tend, - MAMAci::view_2d hetfrz_depostion_nucleation_tend, - MAMAci::view_2d diagnostic_scratch_[]) { - mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState dry_aero = dry_aero_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; - - MAMAci::view_2d diagnostic_scratch[MAMAci::hetro_scratch_]; - for(int i = 0; i < MAMAci::hetro_scratch_; ++i) - diagnostic_scratch[i] = diagnostic_scratch_[i]; - - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - // Set up an atmosphere, surface, diagnostics, pronostics and - // tendencies class. - - haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atmosphere, icol); - haero::Surface surf{}; - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aero, icol); - - const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - - mam4::Diagnostics diags(nlev); - - diags.activation_fraction[accum_idx] = - ekat::subview(factnum, icol, accum_idx); - - diags.activation_fraction[coarse_idx] = - ekat::subview(factnum, icol, coarse_idx); - - // These are the output tendencies from heterogeneous freezing that need - // to be added correctly to the cloud-micorphysics scheme. - diags.hetfrz_immersion_nucleation_tend = - ekat::subview(hetfrz_immersion_nucleation_tend, icol); - diags.hetfrz_contact_nucleation_tend = - ekat::subview(hetfrz_contact_nucleation_tend, icol); - diags.hetfrz_depostion_nucleation_tend = - ekat::subview(hetfrz_depostion_nucleation_tend, icol); - - diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); - diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); - diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); - diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); - diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); - diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); - diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); - diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); - diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); - diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); - diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); - diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); - diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); - diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); - diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); - diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); - diags.na500 = ekat::subview(diagnostic_scratch[18], icol); - diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); - diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); - diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); - diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); - diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); - diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); - diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); - diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); - diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); - diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); - diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); - diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); - diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); - diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); - diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); - diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); - diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); - diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); - diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); - diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); - diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); - diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - diags.stratiform_cloud_fraction = - ekat::subview(diagnostic_scratch[42], icol); - - // assign cloud fraction - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - [=](int klev) { - diags.stratiform_cloud_fraction(klev) = - haero_atm.cloud_fraction(klev); - }); - //------------------------------------------------------------- - // Heterogeneous freezing - // frzimm, frzcnt, frzdep are the outputs of - // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) - //------------------------------------------------------------- - // - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t = 0; // not used - hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, - progs, diags, tends); - }); -} -} // namespace - MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { // Asserts for the runtime or namelist options From 570e91004f037fd83852a95d57c3e5d421957bd7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 24 May 2024 23:00:35 -0700 Subject: [PATCH 129/142] Fixes GPU access inaccessible memory space error with deep_copy --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 67791fe09af..b2bcb7e8f77 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -546,14 +546,7 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); // FIXME: Temporary assignment of nc - // Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [=](int kk) { - nc_inp_to_aci_(icol, kk) = wet_atm_.nc(icol, kk); - }); - }); + Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output From 483db9945f8a219ec36f0cc9aa618ddd7bc4e308 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 24 May 2024 23:24:23 -0700 Subject: [PATCH 130/142] Moves an include from hpp to cpp --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 5 +++++ .../src/physics/mam/eamxx_mam_aci_process_interface.hpp | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b2bcb7e8f77..fc0324823a0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,6 +1,11 @@ #include + +// ACI functions are stored in the following hpp file #include +// For EKAT units package +#include "ekat/util/ekat_units.hpp" + /* ----------------------------------------------------------------- NOTES: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 093dd571dd9..97bc90d3f98 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -7,9 +7,6 @@ // For declaring ACI class derived from atm process class #include -// For EKAT units package -#include "ekat/util/ekat_units.hpp" - // For aerosol configuration #include "mam4xx/aero_config.hpp" From 1db5be888da0a658052319e720b1ba53bda70baf Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 27 May 2024 23:02:21 -0700 Subject: [PATCH 131/142] Adds a new function to copy views --- .../src/physics/mam/eamxx_mam_aci_functions.hpp | 6 +++--- .../mam/eamxx_mam_aci_process_interface.cpp | 5 ++++- .../eamxx/src/physics/mam/mam_coupling.hpp | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 5174eb65236..87018c92952 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -1,8 +1,8 @@ #ifndef EAMXX_MAM_ACI_FUNCTION_HPP #define EAMXX_MAM_ACI_FUNCTION_HPP -#include #include +#include #include namespace scream { @@ -721,6 +721,6 @@ void call_hetfrz_compute_tendencies( }); } } // namespace -} //namespace scream +} // namespace scream -#endif // ifdef EAMXX_MAM_ACI_FUNCTION_HPP \ No newline at end of file +#endif // ifdef EAMXX_MAM_ACI_FUNCTION_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index fc0324823a0..6fd056f2f2d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -20,6 +20,8 @@ this assumption should not cause any issues. FUTURE WORK: 1. MAM4xx submodule should point to MAM4xx main branch 2. Link hetrozenous freezing outputs to microphysics +3. Add postcondition and invariant checks +4. Add variables in output.yaml files of the standalone tests ----------------------------------------------------------------- */ @@ -551,7 +553,8 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); // FIXME: Temporary assignment of nc - Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); + mam_coupling::copy_view_lev_slice(team_policy, wet_atm_.nc, nlev_, // inputs + nc_inp_to_aci_); // output compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 2d89b573a54..a80b41f6c08 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -724,6 +724,23 @@ void compute_wet_mixing_ratios(const Team& team, }); } +// Scream (or EAMxx) can sometimes extend views beyond model levels (nlev) as it uses +// "packs". Following function copies a 2d view till model levels +KOKKOS_INLINE_FUNCTION +void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs + const_view_2d &inp_view, //input view to copy + const int dim, //dimension till view should be copied + view_2d &out_view) { //output view + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, dim), [=](int kk) { + out_view(icol, kk) = inp_view(icol, kk); + }); + }); + } + // Because CUDA C++ doesn't allow us to declare and use constants outside of // KOKKOS_INLINE_FUNCTIONS, we define this macro that allows us to (re)define // these constants where needed within two such functions so we don't define From b00adf27b7a8809a436bdd4c0e1a1f2257b832d8 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 28 May 2024 00:43:22 -0700 Subject: [PATCH 132/142] Updates CMAKE for the aci tests to mimic P3 CMAKE to enable np1 vs npX tests --- .../single-process/mam/aci/CMakeLists.txt | 32 +++++++++++++------ .../tests/single-process/mam/aci/output.yaml | 1 - 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 8c5028828bb..25c83e362ed 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,34 +12,46 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -set (ATM_TIME_STEP 600) +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h set (RUN_T0 2021-10-12-45000) - -# Ensure test input files are present in the data dir -GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) -GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) - +## Copy (and configure) yaml files needed by tests configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} - FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} LABELS mam4_aci physics META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) +# Check tendency calculation +foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) + set (script ${SCREAM_BASE_DIR}/scripts/check-tendencies) + set (fname mam4_aci_standalone_output.INSTANT.nsteps_x1.np${NRANKS}.${RUN_T0}.nc) + set (tname mam4_aci_tend_check_np${NRANKS}) + add_test (NAME ${tname} + COMMAND ${script} -f ${fname} -v qc T_mid -t mam_aci_qc_tend mam_aci_T_mid_tend + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties (${tname} PROPERTIES + LABELS "mam4_aci;physics" + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) +endforeach() + if (SCREAM_ENABLE_BASELINE_TESTS) # Compare one of the output files with the baselines. # Note: one is enough, since we already check that np1 is BFB with npX - set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) - CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) endif() diff --git a/components/eamxx/tests/single-process/mam/aci/output.yaml b/components/eamxx/tests/single-process/mam/aci/output.yaml index 670667576c3..1591d5fe331 100644 --- a/components/eamxx/tests/single-process/mam/aci/output.yaml +++ b/components/eamxx/tests/single-process/mam/aci/output.yaml @@ -11,5 +11,4 @@ Fields: output_control: Frequency: 2 frequency_units: nsteps - MPI Ranks in Filename: true ... From be1417e3825119083107c24f02f16eb39c37f035 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 28 May 2024 01:53:33 -0700 Subject: [PATCH 133/142] Changes after rebase --- .../mam/eamxx_mam_aci_process_interface.cpp | 10 +++---- .../shoc/eamxx_shoc_process_interface.cpp | 27 ++++++++----------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 6fd056f2f2d..4dcc169daa8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -71,10 +71,10 @@ void MAMAci::set_grids( using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers - q_unit.set_string("kg/kg"); + //q_unit.set_string("kg/kg"); auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); + //n_unit.set_string("#/kg"); auto nondim = ekat::units::Units::nondimensional(); @@ -116,9 +116,9 @@ void MAMAci::set_grids( add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); auto m2 = m * m; - m2.set_string("m^2"); + //m2.set_string("m^2"); auto s2 = s * s; - s2.set_string("s^2"); + //s2.set_string("s^2"); // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time // step. Since, we are using SE dycore, itim_old is 1 which is equivalent to @@ -240,7 +240,7 @@ void MAMAci::set_grids( // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); - n_unit.set_string("1(cm^-3 s^-1)"); + //n_unit.set_string("1(cm^-3 s^-1)"); // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 904d7f9672e..15d43c82f7c 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -23,12 +23,6 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids { using namespace ekat::units; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Qunit = kg/kg; - Qunit.set_string("kg/kg"); - auto nondim = Units::nondimensional(); - m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -56,21 +50,22 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids constexpr int ps = Spack::n; - const auto m2 = m*m; - const auto s2 = s*s; + const auto nondim = Units::nondimensional(); + const auto m2 = pow(m,2); + const auto s2 = pow(s,2); // These variables are needed by the interface, but not actually passed to shoc_main. add_field("omega", scalar3d_mid, Pa/s, grid_name, ps); add_field("surf_sens_flux", scalar2d , W/m2, grid_name); add_field("surf_mom_flux", vector2d , N/m2, grid_name); - add_field("surf_evap", scalar2d , kg/m2/s, grid_name); - add_field ("T_mid", scalar3d_mid, K, grid_name, ps); - add_field ("qv", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("surf_evap", scalar2d , kg/(m2*s), grid_name); + add_field ("T_mid", scalar3d_mid, K, grid_name, ps); + add_field ("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); // If TMS is a process, add surface drag coefficient to required fields if (m_params.get("apply_tms", false)) { - add_field("surf_drag_coeff_tms", scalar2d, kg/s/m2, grid_name); + add_field("surf_drag_coeff_tms", scalar2d, kg/(m2*s), grid_name); } // Input variables @@ -84,12 +79,12 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers", ps); add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d , m, grid_name); - add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); + add_field("pbl_height", scalar2d , m, grid_name); + add_field("inv_qc_relvar", scalar3d_mid, pow(kg/kg,2), grid_name, ps); add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); @@ -99,7 +94,7 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { - add_field("vapor_flux", scalar2d, kg/m2/s, grid_name); + add_field("vapor_flux", scalar2d, kg/(m2*s), grid_name); add_field("water_flux", scalar2d, m/s, grid_name); add_field("ice_flux", scalar2d, m/s, grid_name); add_field("heat_flux", scalar2d, W/m2, grid_name); From 2e0ebfbc0a80fd63ef957aa006e63e752f07411c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 30 May 2024 16:22:23 -0700 Subject: [PATCH 134/142] Fixes np1_vs_npX standalone tests and adds mam4xx vars to output --- .../eamxx/cmake/machine-files/compy.cmake | 2 +- .../physics/mam/eamxx_mam_aci_functions.hpp | 1 + .../mam/eamxx_mam_aci_process_interface.cpp | 14 +++-- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 2 +- .../mam/shoc_cldfrac_mam4_aci_p3/output.yaml | 55 ++++++++++++++++- .../output.yaml | 55 ++++++++++++++++- .../output.yaml | 55 ++++++++++++++++- .../mam/shoc_mam4_aci/output.yaml | 55 ++++++++++++++++- .../single-process/mam/aci/CMakeLists.txt | 13 ---- .../tests/single-process/mam/aci/output.yaml | 59 +++++++++++++++++-- 10 files changed, 283 insertions(+), 28 deletions(-) diff --git a/components/eamxx/cmake/machine-files/compy.cmake b/components/eamxx/cmake/machine-files/compy.cmake index 1f156b0546c..e8404d75559 100644 --- a/components/eamxx/cmake/machine-files/compy.cmake +++ b/components/eamxx/cmake/machine-files/compy.cmake @@ -7,4 +7,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) #Compy SLURM specific settings -set(EKAT_MPI_NP_FLAG "-p short -n" CACHE STRING "" FORCE) +set(EKAT_MPI_NP_FLAG "-p short --mpi=pmi2 -n" CACHE STRING "" FORCE) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 87018c92952..e5368f7a360 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -547,6 +547,7 @@ void update_cloud_borne_aerosols( } // Update interstitial aerosols using tendencies - levels +KOKKOS_INLINE_FUNCTION void update_interstitial_aerosols_levs( const haero::ThreadTeam &team, const int nlev, const int icol, const int s_idx, const Real dt, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4dcc169daa8..da50a2282c4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -21,7 +21,11 @@ FUTURE WORK: 1. MAM4xx submodule should point to MAM4xx main branch 2. Link hetrozenous freezing outputs to microphysics 3. Add postcondition and invariant checks -4. Add variables in output.yaml files of the standalone tests +5. Resolve comments about top_lev +6. Avoid using c-style static arrays in ptend and other arrays +7. Use std::string rather than c-strings +8. Remove a Kokkos:fence and combine two kernels while computing w_sec_int_ +9. Fix double counting of tracer advection by modifying SHOC. ----------------------------------------------------------------- */ @@ -145,7 +149,7 @@ void MAMAci::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); + make_layout({ncol_, num_aero_modes, nlev_}, {"ncol", "nmodes", "lev"}); // dry diameter of aerosols [m] add_field("dgnum", scalar4d_layout_mid, m, grid_name); @@ -564,17 +568,16 @@ void MAMAci::run_impl(const double dt) { compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output w_sec_int_); - + Kokkos::fence(); // wait for w_sec_int_ to be computed. compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output tke_); - Kokkos::fence(); // wait for for tke_ to be computed. + Kokkos::fence(); // wait for tke_ to be computed. compute_subgrid_scale_velocities(team_policy, tke_, wsubmin_, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); - Kokkos::fence(); // wait for wsig_ to be computed. // We need dry diameter for only aitken mode compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, @@ -611,6 +614,7 @@ void MAMAci::run_impl(const double dt) { // output kvh_int_); + Kokkos::fence(); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial // aerosols tendencies diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index a8cac4b1a6c..9c0ed4d9798 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -48,7 +48,7 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} #variable required for shoc diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml index f4aec357e9f..2541ce8d9f8 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cld_p3_rrtmgp_output +filename_prefix: shoc_cldfrac_mam4_aci_p3_output Averaging Type: Instant Field Names: # SHOC @@ -38,6 +38,59 @@ Field Names: - qv # SHOC + P3 + RRTMGP - T_mid + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml index a8b22c16ba0..392e159f0a5 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cld_p3_rrtmgp_output +filename_prefix: shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp_output Averaging Type: Instant Field Names: # SHOC @@ -51,6 +51,59 @@ Field Names: - rad_heating_pdel - sfc_flux_lw_dn - sfc_flux_sw_net + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml index a8b22c16ba0..4c9f5ff5d8d 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cld_p3_rrtmgp_output +filename_prefix: shoc_cldfrac_mam4_aci_p3_rrtmgp_output Averaging Type: Instant Field Names: # SHOC @@ -51,6 +51,59 @@ Field Names: - rad_heating_pdel - sfc_flux_lw_dn - sfc_flux_sw_net + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml index 590cd86ab7e..34626c99643 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cld_p3_rrtmgp_output +filename_prefix: shoc_mam4_aci_output Averaging Type: Instant Field Names: # SHOC @@ -11,6 +11,59 @@ Field Names: - tke - inv_qc_relvar - pbl_height + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 25c83e362ed..0d5d60ee7d5 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -36,19 +36,6 @@ CompareNCFilesFamilyMpi ( META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) -# Check tendency calculation -foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) - set (script ${SCREAM_BASE_DIR}/scripts/check-tendencies) - set (fname mam4_aci_standalone_output.INSTANT.nsteps_x1.np${NRANKS}.${RUN_T0}.nc) - set (tname mam4_aci_tend_check_np${NRANKS}) - add_test (NAME ${tname} - COMMAND ${script} -f ${fname} -v qc T_mid -t mam_aci_qc_tend mam_aci_T_mid_tend - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties (${tname} PROPERTIES - LABELS "mam4_aci;physics" - FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) -endforeach() - if (SCREAM_ENABLE_BASELINE_TESTS) # Compare one of the output files with the baselines. # Note: one is enough, since we already check that np1 is BFB with npX diff --git a/components/eamxx/tests/single-process/mam/aci/output.yaml b/components/eamxx/tests/single-process/mam/aci/output.yaml index 1591d5fe331..18f64b077cf 100644 --- a/components/eamxx/tests/single-process/mam/aci/output.yaml +++ b/components/eamxx/tests/single-process/mam/aci/output.yaml @@ -5,10 +5,61 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid - - dgnum - + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated + output_control: - Frequency: 2 + Frequency: 1 frequency_units: nsteps ... From a2afdfcb95838b1290a28fccb9edc1d0a7126a0d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 30 May 2024 17:29:42 -0700 Subject: [PATCH 135/142] Adds some TODO comments and minor cleanup --- .../eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp | 1 - .../physics/mam/eamxx_mam_aci_process_interface.cpp | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index e5368f7a360..96e31f4d6c7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -529,7 +529,6 @@ void call_function_dropmixnuc( // Update cloud borne aerosols void update_cloud_borne_aerosols( - haero::ThreadTeamPolicy team_policy, const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, // output mam_coupling::AerosolState &dry_aero) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index da50a2282c4..063535ccdab 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -26,6 +26,14 @@ FUTURE WORK: 7. Use std::string rather than c-strings 8. Remove a Kokkos:fence and combine two kernels while computing w_sec_int_ 9. Fix double counting of tracer advection by modifying SHOC. +10. A git issue for computing top_lev, moving liq_cldfrac in ACI and using TKE directly +11.Replace [=] with [&] +12. Remove all set_string and use pow +13. use "aitken_dry_dia = ekat::subview_1(dgnum,mam4::ModeIndex::Aitken);" +14. Merge kernels so that we are not calling one from another +15. Use "TeamVectorRange" instead of thread +16. improve the way qqcw is populated +17.delete fence mentioned by Luca ----------------------------------------------------------------- */ @@ -663,7 +671,7 @@ void MAMAci::run_impl(const double dt) { //--------------------------------------------------------------- // Update cloud borne aerosols - update_cloud_borne_aerosols(team_policy, qqcw_fld_work_, nlev_, + update_cloud_borne_aerosols(qqcw_fld_work_, nlev_, // output dry_aero_); From 49db86d21287a5dc03754a8a28ae8d0b55118251 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 30 May 2024 21:31:36 -0700 Subject: [PATCH 136/142] Changes TeamThreadRange to TeamVectorRange; [=] to [&]; removes aitken dia func with deep_copy --- .../physics/mam/eamxx_mam_aci_functions.hpp | 51 +++++-------------- .../mam/eamxx_mam_aci_process_interface.cpp | 32 ++++-------- 2 files changed, 23 insertions(+), 60 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 96e31f4d6c7..a11d9e066d7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -23,12 +23,12 @@ void compute_w0_and_rho(const haero::ThreadTeam &team, // Gas constant for dry air [J/(kg*K) or J/Kg/K] static constexpr auto rair = C::Rair; Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { w0(icol, kk) = 0; rho(icol, kk) = -999.0; }); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); }); @@ -80,7 +80,7 @@ void compute_tke_using_w_sec(const haero::ThreadTeam &team, // output MAMAci::view_2d tke) { Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev + 1), + Kokkos::TeamVectorRange(team, 0u, nlev + 1), KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); } void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, @@ -104,13 +104,13 @@ void compute_subgrid_scale_velocities( // More refined computation of sub-grid vertical velocity // Set to be zero at the surface by initialization. Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { wsub(icol, kk) = wsubmin; wsubice(icol, kk) = 0.001; wsig(icol, kk) = 0.001; }); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * (2.0 / 3.0)); wsig(icol, kk) = @@ -135,33 +135,6 @@ void compute_subgrid_scale_velocities( }); } -KOKKOS_INLINE_FUNCTION -void compute_aitken_dry_diameter(const haero::ThreadTeam &team, - const MAMAci::const_view_3d dgnum, - const int icol, const int top_lev, - const int nlev, - // output - MAMAci::view_2d aitken_dry_dia) { - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); - }); -} -void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_3d dgnum, - const int top_lev, const int nlev, - // output - MAMAci::view_2d aitken_dry_dia) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_aitken_dry_diameter(team, dgnum, icol, top_lev, nlev, - // output - aitken_dry_dia); - }); -} - void compute_nucleate_ice_tendencies( const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, const mam_coupling::DryAtmosphere &dry_atmosphere, @@ -246,7 +219,7 @@ void store_liquid_cloud_fraction( // cut-off for cloud amount (ice or liquid) static constexpr auto qsmall = 1e-18; Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { if((qc(icol, kk) + qi(icol, kk)) > qsmall) { cloud_frac(icol, kk) = liqcldf(icol, kk); cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); @@ -281,7 +254,7 @@ void compute_recipical_pseudo_density(const haero::ThreadTeam &team, // output MAMAci::view_2d rpdel) { Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), "Error: pdel should be > 0.\n"); rpdel(icol, kk) = 1 / pdel(icol, kk); @@ -457,8 +430,8 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - [=](int klev) { + Kokkos::TeamVectorRange(team, 0u, mam4::ndrop::pver), + [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -553,7 +526,7 @@ void update_interstitial_aerosols_levs( const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], // output MAMAci::view_2d aero_mr) { - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [=](int kk) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; }); } @@ -700,8 +673,8 @@ void call_hetfrz_compute_tendencies( // assign cloud fraction Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - [=](int klev) { + Kokkos::TeamVectorRange(team, 0u, mam4::ndrop::pver), + [&](int klev) { diags.stratiform_cloud_fraction(klev) = haero_atm.cloud_fraction(klev); }); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 063535ccdab..c5d6f5eb204 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -26,13 +26,10 @@ FUTURE WORK: 7. Use std::string rather than c-strings 8. Remove a Kokkos:fence and combine two kernels while computing w_sec_int_ 9. Fix double counting of tracer advection by modifying SHOC. -10. A git issue for computing top_lev, moving liq_cldfrac in ACI and using TKE directly -11.Replace [=] with [&] -12. Remove all set_string and use pow -13. use "aitken_dry_dia = ekat::subview_1(dgnum,mam4::ModeIndex::Aitken);" +10. A git issue for computing top_lev, moving liq_cldfrac in ACI and using TKE +directly 14. Merge kernels so that we are not calling one from another -15. Use "TeamVectorRange" instead of thread -16. improve the way qqcw is populated +16. improve the way qqcw is populated 17.delete fence mentioned by Luca ----------------------------------------------------------------- */ @@ -83,10 +80,7 @@ void MAMAci::set_grids( using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers - //q_unit.set_string("kg/kg"); - - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - //n_unit.set_string("#/kg"); + auto n_unit = 1 / kg; // units of number mixing ratios of tracers auto nondim = ekat::units::Units::nondimensional(); @@ -127,10 +121,8 @@ void MAMAci::set_grids( // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); - auto m2 = m * m; - //m2.set_string("m^2"); - auto s2 = s * s; - //s2.set_string("s^2"); + auto m2 = pow(m, 2); + auto s2 = pow(s, 2); // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time // step. Since, we are using SE dycore, itim_old is 1 which is equivalent to @@ -252,8 +244,7 @@ void MAMAci::set_grids( // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); - //n_unit.set_string("1(cm^-3 s^-1)"); - // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] + // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); @@ -586,11 +577,10 @@ void MAMAci::run_impl(const double dt) { // output wsub_, wsubice_, wsig_); - // We need dry diameter for only aitken mode - compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, - // output - aitken_dry_dia_); + Kokkos::deep_copy( + aitken_dry_dia_, + ekat::subview_1(dgnum_, static_cast(mam4::ModeIndex::Aitken))); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -622,7 +612,7 @@ void MAMAci::run_impl(const double dt) { // output kvh_int_); - Kokkos::fence(); + Kokkos::fence(); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial // aerosols tendencies From f8a19928dc24a269ac24c85523f09eeea055cc0a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 31 May 2024 22:32:02 -0700 Subject: [PATCH 137/142] GPU fixes: changes array of views to views in Kokkos loop --- .../physics/mam/eamxx_mam_aci_functions.hpp | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index a11d9e066d7..6c9499da5ce 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -429,8 +429,9 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays + constexpr auto pver = mam4::ndrop::pver; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamVectorRange(team, 0u, pver), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; @@ -522,12 +523,11 @@ void update_cloud_borne_aerosols( KOKKOS_INLINE_FUNCTION void update_interstitial_aerosols_levs( const haero::ThreadTeam &team, const int nlev, const int icol, - const int s_idx, const Real dt, - const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + const Real dt, const MAMAci::view_2d ptend_view, // output MAMAci::view_2d aero_mr) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { - aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; + aero_mr(icol, kk) += ptend_view(icol, kk) * dt; }); } @@ -548,12 +548,13 @@ void update_interstitial_aerosols( auto aero_mmr = dry_aero.int_aero_mmr[m][a]; if(aero_mmr.data()) { + const auto ptend_view = ptend_q[s_idx]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, - ptend_q, + update_interstitial_aerosols_levs(team, nlev, icol, dt, + ptend_view, // output aero_mmr); }); @@ -563,12 +564,13 @@ void update_interstitial_aerosols( } auto aero_nmr = dry_aero.int_aero_nmr[m]; // number mixing ratio for mode "m" + const auto ptend_view = ptend_q[s_idx]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, - ptend_q, + update_interstitial_aerosols_levs(team, nlev, icol, dt, + ptend_view, // output aero_nmr); }); @@ -672,8 +674,9 @@ void call_hetfrz_compute_tendencies( ekat::subview(diagnostic_scratch[42], icol); // assign cloud fraction + constexpr auto pver = mam4::ndrop::pver; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamVectorRange(team, 0u, pver), [&](int klev) { diags.stratiform_cloud_fraction(klev) = haero_atm.cloud_fraction(klev); From 2e4832c2a05d939a2e5c50b5915f2a9b2b05ed98 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 1 Jun 2024 11:25:14 -0700 Subject: [PATCH 138/142] Fixes warnings and changes [=] to [&] --- components/eamxx/src/physics/mam/mam_coupling.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index a80b41f6c08..ed156a9de89 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -735,7 +735,7 @@ void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, dim), [=](int kk) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, dim), [&](int kk) { out_view(icol, kk) = inp_view(icol, kk); }); }); @@ -851,7 +851,6 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; @@ -884,7 +883,6 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; From 2beef5c8f4a892b0182cc37f58662e4a2dbde3fc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 4 Jun 2024 12:25:18 -0700 Subject: [PATCH 139/142] Fixes LABLES of the tests to change mam to mam4_aci for time reporting --- .../physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt | 4 ++-- .../CMakeLists.txt | 4 ++-- .../mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt | 4 ++-- .../physics_only/mam/shoc_mam4_aci/CMakeLists.txt | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt index 946d68cb750..856ef3ca162 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt @@ -6,7 +6,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LIBS shoc cld_fraction p3 mam - LABELS shoc cld p3 physics mam + LABELS shoc cld p3 physics mam4_aci MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -36,6 +36,6 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS shoc cld p3 physics PEM mam + LABELS shoc cld p3 physics PEM mam4_aci META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt index c956df2b8fd..3f9aeae73fd 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt @@ -6,7 +6,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LIBS shoc cld_fraction p3 scream_rrtmgp mam - LABELS shoc cld p3 rrtmgp physics mam + LABELS shoc cld p3 rrtmgp physics mam4_aci MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -36,6 +36,6 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS shoc cld p3 rrtmgp physics PEM mam + LABELS shoc cld p3 rrtmgp physics PEM mam4_aci META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt index bb30c120d38..cdd564c73a8 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt @@ -6,7 +6,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LIBS shoc cld_fraction p3 scream_rrtmgp mam - LABELS shoc cld p3 rrtmgp physics mam + LABELS shoc cld p3 rrtmgp physics mam4_aci MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -36,6 +36,6 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS shoc cld p3 rrtmgp physics PEM mam + LABELS shoc cld p3 rrtmgp physics PEM mam4_aci META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt index af6fff36b0a..69a154223fd 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt @@ -6,7 +6,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LIBS shoc mam - LABELS shoc physics mam + LABELS shoc physics mam4_aci MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -36,6 +36,6 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS shoc physics mam + LABELS shoc physics mam4_aci META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) From 699a486523b78d60753be78c2d9bef9f908af0d4 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 4 Jun 2024 21:35:04 -0700 Subject: [PATCH 140/142] Fixed a layout and Clang format --- .../physics/mam/eamxx_mam_aci_functions.hpp | 28 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.cpp | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 6c9499da5ce..9fedb64555e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -431,8 +431,7 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays constexpr auto pver = mam4::ndrop::pver; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, pver), - [&](int klev) { + Kokkos::TeamVectorRange(team, 0u, pver), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -521,11 +520,12 @@ void update_cloud_borne_aerosols( // Update interstitial aerosols using tendencies - levels KOKKOS_INLINE_FUNCTION -void update_interstitial_aerosols_levs( - const haero::ThreadTeam &team, const int nlev, const int icol, - const Real dt, const MAMAci::view_2d ptend_view, - // output - MAMAci::view_2d aero_mr) { +void update_interstitial_aerosols_levs(const haero::ThreadTeam &team, + const int nlev, const int icol, + const Real dt, + const MAMAci::view_2d ptend_view, + // output + MAMAci::view_2d aero_mr) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { aero_mr(icol, kk) += ptend_view(icol, kk) * dt; }); @@ -569,8 +569,7 @@ void update_interstitial_aerosols( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, dt, - ptend_view, + update_interstitial_aerosols_levs(team, nlev, icol, dt, ptend_view, // output aero_nmr); }); @@ -675,12 +674,11 @@ void call_hetfrz_compute_tendencies( // assign cloud fraction constexpr auto pver = mam4::ndrop::pver; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, pver), - [&](int klev) { - diags.stratiform_cloud_fraction(klev) = - haero_atm.cloud_fraction(klev); - }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, 0u, pver), + [&](int klev) { + diags.stratiform_cloud_fraction(klev) = + haero_atm.cloud_fraction(klev); + }); //------------------------------------------------------------- // Heterogeneous freezing // frzimm, frzcnt, frzdep are the outputs of diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index c5d6f5eb204..25274567902 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -149,7 +149,7 @@ void MAMAci::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"ncol", "nmodes", "lev"}); + make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); // dry diameter of aerosols [m] add_field("dgnum", scalar4d_layout_mid, m, grid_name); From 3f01e52f8237dfd673603c23a51e099180833553 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 5 Jun 2024 07:14:40 -0700 Subject: [PATCH 141/142] Adds a cime test for aci --- cime_config/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/tests.py b/cime_config/tests.py index 3223ae10804..87249990a59 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -690,6 +690,7 @@ "time" : "01:00:00", "tests" : ( "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", ) }, From ea3a693352fb2afb4042bfa053701c0c29c1a4d0 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 12 Jun 2024 11:41:54 -0600 Subject: [PATCH 142/142] Fix uninitialized variable and a few Cuda compiler warnings. --- .../mam/eamxx_mam_aci_process_interface.cpp | 1 + ...mxx_mam_microphysics_process_interface.cpp | 34 +++++++++++-------- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 25274567902..8f9552537ab 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -301,6 +301,7 @@ void MAMAci::initialize_impl(const RunType run_type) { wet_atm_.ni = get_field_in("ni").get_view(); // store rest fo the atm fields in dry_atm_in + dry_atm_.z_surf = 0; dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 6ae36835302..7ea2adfddb6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -306,6 +306,12 @@ void MAMMicrophysics::run_impl(const double dt) { // allocation perspective auto o3_col_dens = buffer_.scratch[8]; + const_view_1d &col_latitudes = col_latitudes_; + mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + mam_coupling::AerosolState &dry_aero = dry_aero_; + mam4::mo_photo::PhotoTableData &photo_table = photo_table_; + const int nlev = nlev_; + const Config &config = config_; // FIXME: read relevant linoz climatology data from file(s) based on time // FIXME: read relevant chlorine loading data from file based on time @@ -314,21 +320,21 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index - Real col_lat = col_latitudes_(icol); // column latitude (degrees?) + Real col_lat = col_latitudes(icol); // column latitude (degrees?) // fetch column-specific atmosphere state data - auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); - auto z_iface = ekat::subview(dry_atm_.z_iface, icol); - Real phis = dry_atm_.phis(icol); + auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + auto z_iface = ekat::subview(dry_atm.z_iface, icol); + Real phis = dry_atm.phis(icol); // set surface state data haero::Surface sfc{}; // fetch column-specific subviews into aerosol prognostics - mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); + mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero, icol); // set up diagnostics - mam4::Diagnostics diags(nlev_); + mam4::Diagnostics diags(nlev); // calculate o3 column densities (first component of col_dens in Fortran code) auto o3_col_dens_i = ekat::subview(o3_col_dens, icol); @@ -347,7 +353,7 @@ void MAMMicrophysics::run_impl(const double dt) { mam4::ColumnView lwc; // FIXME: liquid water cloud content: where do we get this? mam4::mo_photo::table_photo(photo_rates, atm.pressure, atm.hydrostatic_dp, atm.temperature, o3_col_dens_i, zenith_angle, surf_albedo, lwc, - atm.cloud_fraction, esfact, photo_table_, photo_work_arrays); + atm.cloud_fraction, esfact, photo_table, photo_work_arrays); // compute external forcings at time t(n+1) [molecules/cm^3/s] constexpr int extcnt = mam4::gas_chemistry::extcnt; @@ -356,7 +362,7 @@ void MAMMicrophysics::run_impl(const double dt) { mam4::mo_setext::extfrc_set(forcings, extfrc); // compute aerosol microphysics on each vertical level within this column - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int k) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [&](const int k) { constexpr int num_modes = mam4::AeroConfig::num_modes(); constexpr int gas_pcnst = mam_coupling::gas_pcnst(); @@ -429,7 +435,7 @@ void MAMMicrophysics::run_impl(const double dt) { constexpr int indexm = 0; // FIXME: index of xhnm in invariants array (??) Real cldnum = 0.0; // FIXME: droplet number concentration: where do we get this? setsox_single_level(loffset, dt, pmid, pdel, temp, mbar, lwc(k), - cldfrac, cldnum, invariants[indexm], config_.setsox, vmrcw, vmr); + cldfrac, cldnum, invariants[indexm], config.setsox, vmrcw, vmr); // calculate aerosol water content using water uptake treatment // * dry and wet diameters [m] @@ -442,7 +448,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config_.amicphys, step_, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, @@ -467,15 +473,15 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_o3_clim(icol, k), linoz_t_clim(icol, k), linoz_o3col_clim(icol, k), linoz_PmL_clim(icol, k), linoz_dPmL_dO3(icol, k), linoz_dPmL_dT(icol, k), linoz_dPmL_dO3col(icol, k), linoz_cariolle_psc(icol, k), - chlorine_loading, config_.linoz.psc_T, vmr[o3_ndx], + chlorine_loading, config.linoz.psc_T, vmr[o3_ndx], do3_linoz, do3_linoz_psc, ss_o3, o3col_du_diag, o3clim_linoz_diag, zenith_angle_degrees); // update source terms above the ozone decay threshold - if (k > nlev_ - config_.linoz.o3_lbl - 1) { + if (k > nlev - config.linoz.o3_lbl - 1) { Real do3_mass; // diagnostic, not needed - mam4::lin_strat_chem::lin_strat_sfcsink_kk(dt, pdel, vmr[o3_ndx], config_.linoz.o3_sfc, - config_.linoz.o3_tau, do3_mass); + mam4::lin_strat_chem::lin_strat_sfcsink_kk(dt, pdel, vmr[o3_ndx], config.linoz.o3_sfc, + config.linoz.o3_tau, do3_mass); } // ... check for negative values and reset to zero diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index ed156a9de89..3a001146882 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -726,7 +726,7 @@ void compute_wet_mixing_ratios(const Team& team, // Scream (or EAMxx) can sometimes extend views beyond model levels (nlev) as it uses // "packs". Following function copies a 2d view till model levels -KOKKOS_INLINE_FUNCTION +inline void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs const_view_2d &inp_view, //input view to copy const int dim, //dimension till view should be copied