From ecc0b523ccd5d6ccdea50dc7574b9e5b6c4893c5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 19 Feb 2024 19:48:50 -0800 Subject: [PATCH 001/477] Minimal set of additions for drydep interface --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- ...x_mam_dry_deposition_process_interface.cpp | 93 +++++++++++++++++++ ...x_mam_dry_deposition_process_interface.hpp | 64 +++++++++++++ .../eamxx/src/physics/register_physics.hpp | 2 + 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index a4e095bf5f2a..fc48acce3f23 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(mam eamxx_mam_microphysics_process_interface.cpp - eamxx_mam_optics_process_interface.cpp) + eamxx_mam_optics_process_interface.cpp + eamxx_mam_dry_deposition_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx_proj) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp new file mode 100644 index 000000000000..7a8adf5d087f --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -0,0 +1,93 @@ +#include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" + +/* +Future work: +Wirte comments +write in/outs for all variables clearly +*/ + +namespace scream { + +// ========================================================================================= +MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ========================================================================================= +void MAMDryDep::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + + // Layout for 2D (2d horiz) variable defined at mid-level and + // interfaces + const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + + +} + +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMDryDep::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMDryDep."); +} + +// ========================================================================================= +void MAMDryDep::initialize_impl(const RunType run_type) { + // Gather runtime options + //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + +} + +// ========================================================================================= +void MAMDryDep::run_impl(const double dt) { + + std::cout << "End of derydep run" << std::endl; +} + +// ========================================================================================= +} // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp new file mode 100644 index 000000000000..7692205dbafb --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -0,0 +1,64 @@ +#ifndef EAMXX_MAM_DRYDEP_HPP +#define EAMXX_MAM_DRYDEP_HPP + +#include + +// For declaring wetscav class derived from atm process class +#include + +// For MAM4 aerosol configuration +#include + +// For component name +#include + +namespace scream { + +// The process responsible for handling MAM4 dry deposition. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMDryDep final : public scream::AtmosphereProcess { + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + public: + // Constructor + MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_dry_deposition"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMDryDep + +} // namespace scream + +#endif // EAMXX_MAM_DRYDEP_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 329b7022756c..decab942c849 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -26,6 +26,7 @@ #ifdef EAMXX_HAS_MAM #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" +#include "physics/mam/eamxx_mam_dry_deposition_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -62,6 +63,7 @@ inline void register_physics () { #ifdef EAMXX_HAS_MAM proc_factory.register_product("mam4_micro",&create_atmosphere_process); proc_factory.register_product("mam4_optics",&create_atmosphere_process); + proc_factory.register_product("mam4_drydep",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 0f58dcaae2cd35beec2dc0533f2ca130d625dbd1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 20 Feb 2024 11:35:00 -0800 Subject: [PATCH 002/477] Adds a simple minimal test and it works --- .../eamxx_mam_dry_deposition_process_interface.cpp | 11 ++++------- .../eamxx_mam_dry_deposition_process_interface.hpp | 2 -- components/eamxx/src/physics/register_physics.hpp | 2 +- components/eamxx/tests/uncoupled/CMakeLists.txt | 2 +- .../tests/uncoupled/mam4/{ => drydep}/CMakeLists.txt | 5 +---- .../tests/uncoupled/mam4/{ => drydep}/input.yaml | 10 +--------- .../tests/uncoupled/mam4/{ => drydep}/output.yaml | 6 +----- 7 files changed, 9 insertions(+), 29 deletions(-) rename components/eamxx/tests/uncoupled/mam4/{ => drydep}/CMakeLists.txt (80%) rename components/eamxx/tests/uncoupled/mam4/{ => drydep}/input.yaml (63%) rename components/eamxx/tests/uncoupled/mam4/{ => drydep}/output.yaml (59%) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 7a8adf5d087f..f27e5d6a1ee4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -29,7 +29,7 @@ void MAMDryDep::set_grids( auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -49,8 +49,8 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - - + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] } // ========================================================================================= @@ -79,15 +79,12 @@ void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { void MAMDryDep::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - - } // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - std::cout << "End of derydep run" << std::endl; } // ========================================================================================= -} // namespace scream \ No newline at end of file +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 7692205dbafb..47be070c4228 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -1,8 +1,6 @@ #ifndef EAMXX_MAM_DRYDEP_HPP #define EAMXX_MAM_DRYDEP_HPP -#include - // For declaring wetscav class derived from atm process class #include diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index decab942c849..bfb3428b7afc 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -26,7 +26,7 @@ #ifdef EAMXX_HAS_MAM #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" -#include "physics/mam/eamxx_mam_dry_deposition_interface.hpp" +#include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" diff --git a/components/eamxx/tests/uncoupled/CMakeLists.txt b/components/eamxx/tests/uncoupled/CMakeLists.txt index dfac92140396..b16d6ac8ccc1 100644 --- a/components/eamxx/tests/uncoupled/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/CMakeLists.txt @@ -21,7 +21,7 @@ if (NOT SCREAM_BASELINES_ONLY) # Once the mam4xx aerosol microphysics AtmosphereProcess is running, the # corresponding test here needs to be reworked with valid aerosol # initial conditions. - #add_subdirectory(mam4) + add_subdirectory(mam4/drydep) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4/drydep/CMakeLists.txt similarity index 80% rename from components/eamxx/tests/uncoupled/mam4/CMakeLists.txt rename to components/eamxx/tests/uncoupled/mam4/drydep/CMakeLists.txt index 18448d025667..17cee4b0fe03 100644 --- a/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt +++ b/components/eamxx/tests/uncoupled/mam4/drydep/CMakeLists.txt @@ -1,10 +1,7 @@ include (ScreamUtils) -# This test needs to be reworked for microphysics -- currently it's still using -# input for nucleation. - # Create the test -CreateADUnitTest(mam4_microphysics_standalone +CreateADUnitTest(mam4_drydep_standalone LABELS mam4 physics LIBS mam MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} diff --git a/components/eamxx/tests/uncoupled/mam4/input.yaml b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml similarity index 63% rename from components/eamxx/tests/uncoupled/mam4/input.yaml rename to components/eamxx/tests/uncoupled/mam4/drydep/input.yaml index 80571b614f0d..c6499e85b508 100644 --- a/components/eamxx/tests/uncoupled/mam4/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml @@ -9,9 +9,7 @@ time_stepping: number_of_steps: ${NUM_STEPS} atmosphere_processes: - atm_procs_list: [mam4_micro] - mam4_micro: - compute_tendencies: [q_aitken_so4, n_aitken, q_h2so4] + atm_procs_list: [mam4_drydep] grids_manager: Type: Mesh Free @@ -24,13 +22,7 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} - q_aitken_so4: 0.0 - n_aitken: 0.0 - q_h2so4: 1.9186478479542893e-011 # 0.65e-10 is from namelist, but this is what gets to nucleation - pbl_height: 1100.0 T_mid: 273.0 - p_mid: 1.e5 - qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/uncoupled/mam4/output.yaml b/components/eamxx/tests/uncoupled/mam4/drydep/output.yaml similarity index 59% rename from components/eamxx/tests/uncoupled/mam4/output.yaml rename to components/eamxx/tests/uncoupled/mam4/drydep/output.yaml index 4e6ad2ac9128..e900bb804340 100644 --- a/components/eamxx/tests/uncoupled/mam4/output.yaml +++ b/components/eamxx/tests/uncoupled/mam4/drydep/output.yaml @@ -1,13 +1,9 @@ %YAML 1.1 --- -filename_prefix: mam4_nucleation_standalone_output +filename_prefix: mam4_drydep_standalone_output Averaging Type: Instant Field Names: - T_mid - - p_mid - - q_aitken_so4 - - n_aitken - - q_h2so4 output_control: Frequency: 1 frequency_units: nsteps From 7eeb63cf2158bbbd90d21da6241d2087cd00f212 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 20 Feb 2024 18:06:00 -0800 Subject: [PATCH 003/477] Adds required variables and the initial condition file --- ...x_mam_dry_deposition_process_interface.cpp | 94 ++++++++++++++++++- ...x_mam_dry_deposition_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 14 +-- .../eamxx/src/share/field/field_tag.hpp | 7 +- components/eamxx/tests/CMakeLists.txt | 1 + .../tests/uncoupled/mam4/drydep/input.yaml | 11 ++- 6 files changed, 116 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index f27e5d6a1ee4..fb71e4708c52 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -29,6 +29,8 @@ void MAMDryDep::set_grids( auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); + auto m3 = m * m * m; // meter cubed + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); @@ -41,16 +43,99 @@ void MAMDryDep::set_grids( // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // Layout for 2D (2d horiz) variable defined at mid-level and - // interfaces - const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + const int num_aero_modes = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_layout_mid{ + {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa] + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + add_field("obklen", scalar2d_layout, m, grid_name); + add_field("surfric", scalar2d_layout, m / s, grid_name); + + auto nondim = ekat::units::Units::nondimensional(); + add_field("landfrac", scalar2d_layout, nondim, grid_name); + add_field("icefrac", scalar2d_layout, nondim, grid_name); + add_field("ocnfrac", scalar2d_layout, nondim, grid_name); + add_field("fv", scalar2d_layout, m / s, grid_name); + add_field("ram1", scalar2d_layout, s / m, grid_name); + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + // printf("%s \n", int_nmr_field_name); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + + /*dgncur_awet, wetdens, obklen, surfric, + cam_in%landfrac, cam_in%icefrac, cam_in%ocnfrac, & cam_in%fv, + cam_in%ram1,*/ + + /*state%t, state%pmid, state%pint, state%pdel, & + state%q, dgncur_awet, wetdens, qqcw, obklen, surfric, + cam_in%landfrac, cam_in%icefrac, cam_in%ocnfrac, & cam_in%fv, cam_in%ram1, + ztodt, cam_out, ptend */ } // ========================================================================================= @@ -84,6 +169,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMDryDep::run_impl(const double dt) { std::cout << "End of derydep run" << std::endl; + + + } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 47be070c4228..7b054657a18f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -1,7 +1,7 @@ #ifndef EAMXX_MAM_DRYDEP_HPP #define EAMXX_MAM_DRYDEP_HPP -// For declaring wetscav class derived from atm process class +// For declaring dry deposition class derived from atm process class #include // For MAM4 aerosol configuration diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index cc627643914e..830743cd3f63 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -102,12 +102,12 @@ const char* aero_species_name(const int species_id) { KOKKOS_INLINE_FUNCTION const char* gas_species_name(const int gas_id) { static const char *species_names[num_aero_gases()] = { - "o3", - "h2o2", - "h2so4", - "so2", - "dms", - "soag" + "O3", + "H2O2", + "H2SO4", + "SO2", + "DMS", + "SOAG" }; return species_names[gas_id]; } @@ -245,7 +245,7 @@ const char* cld_aero_mmr_field_name(const int mode, const int species) { KOKKOS_INLINE_FUNCTION const char* gas_mmr_field_name(const int gas) { if (!gas_mmr_names(gas)[0]) { - concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); + concat_2_strings("", gas_species_name(gas), gas_mmr_names(gas)); } return const_cast(gas_mmr_names(gas)); } diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index a96f781a465c..2e8014273b95 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -40,7 +40,8 @@ enum class FieldTag { LongWaveBand, LongWaveGpoint, IsccpTau, - IsccpPrs + IsccpPrs, + num_modes }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -65,6 +66,7 @@ namespace ShortFieldTagsNames { constexpr auto LWGPT = FieldTag::LongWaveGpoint; constexpr auto ISCCPTAU = FieldTag::IsccpTau; constexpr auto ISCCPPRS = FieldTag::IsccpPrs; + constexpr auto NMODES = FieldTag::num_modes; } inline std::string e2str (const FieldTag ft) { @@ -117,6 +119,9 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::IsccpPrs: name = "ISCCPPRS"; break; + case FieldTag::num_modes: + name = "num_modes"; + break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index ab168e67e79f..6df2d89dd5c4 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -12,6 +12,7 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) # Initial condition files used in the tests set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") + set(EAMxx_tests_IC_FILE_MAM "scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc") set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") # Testing individual atm processes diff --git a/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml index c6499e85b508..3d76ca948282 100644 --- a/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml @@ -21,8 +21,17 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM} T_mid: 273.0 + dgncur_awet: 1e-5 + wetdens: 1e-5 + obklen: 1e-5 + surfric: 1e-5 + landfrac: 1.0 + icefrac: 0.0 + ocnfrac: 0.0 + fv: 1e-5 + ram1: 1e-5 # The parameters for I/O control Scorpio: From 2dd3d73b29762be5c0eb2dd84a4e2b964dc81a18 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 20 Feb 2024 21:39:04 -0800 Subject: [PATCH 004/477] Adds variables to the init and run functions --- ...x_mam_dry_deposition_process_interface.cpp | 130 ++++++++++++++++-- ...x_mam_dry_deposition_process_interface.hpp | 55 ++++++++ 2 files changed, 176 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index fb71e4708c52..1eb1aa95b57c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -74,6 +74,10 @@ void MAMDryDep::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); add_field("obklen", scalar2d_layout, m, grid_name); @@ -127,15 +131,6 @@ void MAMDryDep::set_grids( add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - - /*dgncur_awet, wetdens, obklen, surfric, - cam_in%landfrac, cam_in%icefrac, cam_in%ocnfrac, & cam_in%fv, - cam_in%ram1,*/ - - /*state%t, state%pmid, state%pint, state%pdel, & - state%q, dgncur_awet, wetdens, qqcw, obklen, surfric, - cam_in%landfrac, cam_in%icefrac, cam_in%ocnfrac, & cam_in%fv, cam_in%ram1, - ztodt, cam_out, ptend */ } // ========================================================================================= @@ -164,10 +159,127 @@ void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { void MAMDryDep::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = + get_field_in("pseudo_density").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // set up our preprocess functor + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, + dry_atm_, dry_aero_); } // ========================================================================================= void MAMDryDep::run_impl(const double dt) { + +const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + + + /* + + Rough notes: + + tair == T_mid + pmid == p_mid + pint ==p_int + pdel == p_del + + state_q: It can be obtained using dry_aero_ and dry_atm_, there is an example of this in the optics code + qqcw can also be obtained from wet_aero.There is an example in optics code + Feel free to improvise input variables by adding them to the input.yaml file. + + + + Function to call in mam4xx/drydep.hpp: + void aero_model_drydep( + const ThreadTeam &team, + const Real fraction_landuse[DryDeposition::n_land_type] , + const haero::ConstColumnView tair, const haero::ConstColumnView pmid, + const haero::ConstColumnView pint, const haero::ConstColumnView pdel, + const Diagnostics::ColumnTracerView state_q, + const ColumnView dgncur_awet[AeroConfig::num_modes()], + const ColumnView wetdens[AeroConfig::num_modes()], + const Kokkos::View qqcw[aero_model::pcnst], const Real obklen, + const Real ustar, const Real landfrac, const Real icefrac, + const Real ocnfrac, const Real fricvelin, const Real ram1in, + const Diagnostics::ColumnTracerView ptend_q, + bool ptend_lq[aero_model::pcnst], const Real dt, + const ColumnView aerdepdrycw, const ColumnView aerdepdryis, + + const ColumnView rho, + const Kokkos::View vlc_dry[AeroConfig::num_modes()] + [DryDeposition::aerosol_categories], + const Kokkos::View vlc_trb[AeroConfig::num_modes()] + [DryDeposition::aerosol_categories], + const Kokkos::View vlc_grv[AeroConfig::num_modes()] + [DryDeposition::aerosol_categories], + const Kokkos::View dqdt_tmp[aero_model::pcnst]) + + */ + + + std::cout << "End of derydep run" << std::endl; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 7b054657a18f..7ab45d7fd195 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -15,9 +15,19 @@ namespace scream { // The process responsible for handling MAM4 dry deposition. The AD // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { + + using KT = ekat::KokkosTypes; + // number of horizontal columns and vertical levels int ncol_, nlev_; + //Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + // buffer for sotring temporary variables mam_coupling::Buffer buffer_; @@ -55,6 +65,51 @@ class MAMDryDep final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); + team.team_barrier(); + + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMAci::Preprocess + + private: + // pre- and postprocessing scratch pads + Preprocess preprocess_; + }; // MAMDryDep } // namespace scream From 9c7e8f5ba5fe268b3f40164347c5668f6f5ce57f Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 5 Mar 2024 15:00:28 -0700 Subject: [PATCH 005/477] Implement the MAMDryDep::run_impl() function. This is an interface to the mam4::DryDeposition::compute_tendencies() function. This means taking Scream Kokkos views of data and mapping them to mam4 Kokkos views of data. Most of the time it is just a sub-view so memory is not created or destroyed. There were a couple of choices for mapping which Scream data to the mam4 data that need to be reviewed. The resulting tendencies are in a local Kokkos view called tendencies. These need to be made available to Scream. This needs to be compiled and tested on a GPU. --- ...x_mam_dry_deposition_process_interface.cpp | 288 ++++++++++++++---- ...x_mam_dry_deposition_process_interface.hpp | 42 ++- .../mam/impl/compute_water_content.cpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 6 +- .../share/grid/remap/identity_remapper.hpp | 4 +- externals/haero | 2 +- externals/mam4xx | 2 +- 7 files changed, 270 insertions(+), 76 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 1eb1aa95b57c..65f0e280b877 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -1,5 +1,7 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" +#include "mam4xx/drydep.hpp" + /* Future work: Wirte comments @@ -14,6 +16,27 @@ MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. */ + std::string nname = "Obukhov_length"; + if(params.isParameter(nname)) + parameters_.Obukhov_length_ = params.get(nname); + nname = "surface_friction_velocty"; + if(params.isParameter(nname)) + parameters_.surface_friction_velocty_ = params.get(nname); + nname = "land_fraction"; + if(params.isParameter(nname)) + parameters_.land_fraction_ = params.get(nname); + nname = "ice_fraction"; + if(params.isParameter(nname)) + parameters_.ice_fraction_ = params.get(nname); + nname = "ocean_fraction"; + if(params.isParameter(nname)) + parameters_.ocean_fraction_ = params.get(nname); + nname = "friction_velocity"; + if(params.isParameter(nname)) + parameters_.friction_velocity_ = params.get(nname); + nname = "aerodynamical_resistance"; + if(params.isParameter(nname)) + parameters_.aerodynamical_resistance_ = params.get(nname); } // ========================================================================================= @@ -40,22 +63,42 @@ void MAMDryDep::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; + auto nondim = ekat::units::Units::nondimensional(); + + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ - {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + {NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; // mid points + + // Layout for tracers. + const int pcnst = mam4::aero_model::pcnst; + FieldLayout scalar4d_layout_q{ + {COL, LEV, NMODES}, {ncol_, nlev_, pcnst}}; + FieldLayout scalar4d_layout_qqcw_tends{ + {COL, NMODES, LEV}, {ncol_, pcnst, nlev_}}; + + // Memory to format teh wet_aero.int_aero_nmr and wet_aero.int_aero_mmr + // views into the order expected by mam4xx. + add_field("qtracers", scalar4d_layout_q, kg / kg, grid_name); + add_field("d_qtracers_dt", scalar4d_layout_q, kg / kg / s, grid_name); + // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); + // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_interstitial_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); + add_field("Tendencies", scalar4d_layout_qqcw_tends, kg / kg / s, grid_name); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- + add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, @@ -77,13 +120,14 @@ void MAMDryDep::set_grids( add_field( "omega", scalar3d_layout_mid, Pa / s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); - add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + // + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); add_field("obklen", scalar2d_layout, m, grid_name); add_field("surfric", scalar2d_layout, m / s, grid_name); - auto nondim = ekat::units::Units::nondimensional(); add_field("landfrac", scalar2d_layout, nondim, grid_name); add_field("icefrac", scalar2d_layout, nondim, grid_name); add_field("ocnfrac", scalar2d_layout, nondim, grid_name); @@ -92,7 +136,7 @@ void MAMDryDep::set_grids( // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < num_aero_modes; ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, @@ -108,7 +152,7 @@ void MAMDryDep::set_grids( } } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < num_aero_modes; ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); // printf("%s \n", int_nmr_field_name); @@ -130,7 +174,7 @@ void MAMDryDep::set_grids( const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } + } } // ========================================================================================= @@ -169,14 +213,28 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = - get_field_in("pseudo_density").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; dry_atm_.qv = buffer_.qv_dry; dry_atm_.qc = buffer_.qc_dry; dry_atm_.nc = buffer_.nc_dry; dry_atm_.qi = buffer_.qi_dry; dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + + dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); + qtracers_ = get_field_out("qtracers").get_view(); + d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols").get_view(); + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols").get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -223,67 +281,179 @@ void MAMDryDep::initialize_impl(const RunType run_type) { preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); } +namespace { +KOKKOS_INLINE_FUNCTION +void compute_tendencies( + const MAMDryDep::KT::MemberType &team, + const MAMDryDep::Parameters ¶meters, + const mam4::DryDeposition &dry_deposition, + const double dt, + MAMDryDep::view_3d qtracers, + MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::view_2d dgncur_awet[mam_coupling::num_aero_modes()], + const MAMDryDep::view_2d wet_dens[mam_coupling::num_aero_modes()], + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero, + const mam_coupling::AerosolState &wet_aero, + MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_2d aerdepdryis, + MAMDryDep::view_3d tendencies) +{ + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); + const Real t = 0; + const int icol = team.league_rank(); + + const mam4::AeroConfig aero_config; + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + atm.interface_pressure = atm.pressure; + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + mam4::Surface surf; + mam4::Diagnostics diags; + mam4::Tendencies tends; + + diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); + diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); + + for (int i=0; i::get_default_team_policy(ncol, nlev); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { + compute_tendencies(team, parameters, dry_deposition, dt, qtracers, d_qtracers_dt, + dgncur_awet, wet_dens, dry_atm, dry_aero, wet_aero, aerdepdrycw, aerdepdryis, tendencies); + }); +} + +void fill_tracer_views( + const int ncol, + const int nlev, + mam_coupling::DryAtmosphere dry_atm, + mam_coupling::AerosolState dry_aero, + mam_coupling::AerosolState wet_aero, + MAMDryDep::view_3d qtracers) +{ + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlev); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { + const int column = team.league_rank(); + compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, column); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int lev) { + for (int mode = 0; mode < num_aero_modes; ++mode) { + int icnst = mam4::ConvProc::numptrcw_amode(mode); + qtracers(column, lev, icnst) = wet_aero.int_aero_nmr[mode](column,lev); + for (int species = 0; species < num_aero_species; ++species) { + icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); + if (-1 < icnst) { + qtracers(column, lev, icnst) = wet_aero.int_aero_mmr[mode][species](column,lev); + } + } + } + }); + }); +} +} // ========================================================================================= void MAMDryDep::run_impl(const double dt) { -const auto scan_policy = ekat::ExeSpaceUtils< + // Include this but update when scream is updated to use the newer DryDeposition class: + using DryDep = mam4::DryDeposition; // mam4::DryDeposition; + + using MemberType = KT::MemberType; + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + std::cout<<__FILE__<<":"<<__LINE__<<" In MAMDryDep::run_impl"< qqcw[aero_model::pcnst], const Real obklen, - const Real ustar, const Real landfrac, const Real icefrac, - const Real ocnfrac, const Real fricvelin, const Real ram1in, - const Diagnostics::ColumnTracerView ptend_q, - bool ptend_lq[aero_model::pcnst], const Real dt, - const ColumnView aerdepdrycw, const ColumnView aerdepdryis, - - const ColumnView rho, - const Kokkos::View vlc_dry[AeroConfig::num_modes()] - [DryDeposition::aerosol_categories], - const Kokkos::View vlc_trb[AeroConfig::num_modes()] - [DryDeposition::aerosol_categories], - const Kokkos::View vlc_grv[AeroConfig::num_modes()] - [DryDeposition::aerosol_categories], - const Kokkos::View dqdt_tmp[aero_model::pcnst]) + const DryDep::Config process_config; + // Future enhancement: + // This is where we can set the fraction of land use parameters: + // if (do_soilw) { + // const Real *landuse = get_landuse_from_file(do_soilw); + // for (int i=0; i::get_default_team_policy(ncol_, nlev_); + // Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + // dry_deposition.validate( aero_config, team, atm, progs); + // } + // Kokkos::fence(); + + view_2d dgncur_awet[num_aero_modes]; + for (int i=0; i; - +public: + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); +private: // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -34,7 +38,18 @@ class MAMDryDep final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - public: + view_3d qtracers_; + view_3d d_qtracers_dt_; + view_3d dgncur_awet_; + view_3d wet_dens_; + view_3d tendencies_; + + view_2d aerdepdrycw_; + view_2d aerdepdryis_; + +public: + using KT = ekat::KokkosTypes; + // Constructor MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -43,10 +58,10 @@ class MAMDryDep final : public scream::AtmosphereProcess { // -------------------------------------------------------------------------- // The type of subcomponent - AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type() const override { return AtmosphereProcessType::Physics; } // The name of the subcomponent - std::string name() const { return "mam_dry_deposition"; } + std::string name() const override { return "mam_dry_deposition"; } // grid void set_grids( @@ -63,7 +78,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { void run_impl(const double dt) override; // Finalize - void finalize_impl(){/*Do nothing*/}; + void finalize_impl() override {/*Do nothing*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. @@ -106,12 +121,21 @@ class MAMDryDep final : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess + struct Parameters { + Real Obukhov_length_ = 0.20723257141035126e+03; + Real surface_friction_velocty_ = 0.39900396673305327; + Real land_fraction_ = 0.1; + Real ice_fraction_ = 0.6; + Real ocean_fraction_ = 0.3; + Real friction_velocity_ = 0.46637129718055864; + Real aerodynamical_resistance_ = 0.91147859222259044e+02; + }; private: // pre- and postprocessing scratch pads Preprocess preprocess_; - + Parameters parameters_; }; // MAMDryDep } // namespace scream -#endif // EAMXX_MAM_DRYDEP_HPP \ No newline at end of file +#endif // EAMXX_MAM_DRYDEP_HPP diff --git a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp index a31d81f2408a..673fe06292e1 100644 --- a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp +++ b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp @@ -23,7 +23,7 @@ void compute_water_content(const mam4::Prognostics &progs, int k, // extract aerosol tracers for this level into state_q, which is needed // for computing dry aerosol properties below // FIXME: we should eliminate this index translation stuff - constexpr int nvars = mam4::water_uptake::nvars; + constexpr int nvars = mam4::aero_model::pcnst; Real state_q[nvars]; // aerosol tracers for level k for (int imode = 0; imode < num_modes; ++imode) { int la, lc; // interstitial and cloudborne indices within state_q diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 830743cd3f63..388ff9e27329 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -512,6 +512,7 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, "cldfrac not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, "w_updraft not defined for dry atmosphere state!"); + haero::ConstColumnView interface_pressure; return haero::Atmosphere(mam4::nlev, ekat::subview(dry_atm.T_mid, column_index), ekat::subview(dry_atm.p_mid, column_index), @@ -522,6 +523,7 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, ekat::subview(dry_atm.ni, column_index), ekat::subview(dry_atm.z_mid, column_index), ekat::subview(dry_atm.p_del, column_index), + interface_pressure, ekat::subview(dry_atm.cldfrac, column_index), ekat::subview(dry_atm.w_updraft, column_index), dry_atm.pblh(column_index)); @@ -720,7 +722,7 @@ void compute_wet_mixing_ratios(const Team& team, const auto PC = mam4::ModeIndex::PrimaryCarbon; \ const auto NoMode = mam4::ModeIndex::None; \ static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ @@ -817,7 +819,6 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; @@ -850,7 +851,6 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; diff --git a/components/eamxx/src/share/grid/remap/identity_remapper.hpp b/components/eamxx/src/share/grid/remap/identity_remapper.hpp index 047bd90ef835..d281ce4bc6e7 100644 --- a/components/eamxx/src/share/grid/remap/identity_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/identity_remapper.hpp @@ -63,7 +63,7 @@ class IdentityRemapper : public AbstractRemapper return src_layout; } - void register_field_from_src (const field_type& src) { + void register_field_from_src (const field_type& src) override { EKAT_REQUIRE_MSG (m_aliasing!=SrcAliasTgt, "Error! Makes no sense to register from src and ask that src alias tgt.\n"); if (m_aliasing==TgtAliasSrc) { @@ -72,7 +72,7 @@ class IdentityRemapper : public AbstractRemapper AbstractRemapper::register_field_from_src(src); } } - void register_field_from_tgt (const field_type& tgt) { + void register_field_from_tgt (const field_type& tgt) override { EKAT_REQUIRE_MSG (m_aliasing!=TgtAliasSrc, "Error! Makes no sense to register from tgt and ask that tgt alias src.\n"); if (m_aliasing==SrcAliasTgt) { diff --git a/externals/haero b/externals/haero index ba86d80dc740..d3a953d43979 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit ba86d80dc740dc2dbb8220ab5e501f2547355fa0 +Subproject commit d3a953d43979aa2fc3ae6f72ebc899e9a756ad89 diff --git a/externals/mam4xx b/externals/mam4xx index 209126e50476..d43fe41047a8 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 209126e50476c1d7e0aafd77cec4caea1c56a880 +Subproject commit d43fe41047a8936d2649b8d4087bb820c04919f7 From a6b550cef2fa99dba45f891caad877b0d8f639a8 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 6 Mar 2024 11:00:54 -0700 Subject: [PATCH 006/477] Forgot to add updated test file. --- components/eamxx/tests/uncoupled/mam4/drydep/input.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml index 3d76ca948282..d36e6fe8a55c 100644 --- a/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/drydep/input.yaml @@ -32,6 +32,8 @@ initial_conditions: ocnfrac: 0.0 fv: 1e-5 ram1: 1e-5 + w_updraft: 1e-5 + pbl_height: 0.0 # The parameters for I/O control Scorpio: From 9f59dcb2dbdd4497a1c23b5195488917e0ec76b0 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 6 Mar 2024 12:30:12 -0700 Subject: [PATCH 007/477] Fixes for Cuda build. --- ...x_mam_dry_deposition_process_interface.cpp | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 65f0e280b877..03d44474bc86 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -154,7 +154,6 @@ void MAMDryDep::set_grids( // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios for(int m = 0; m < num_aero_modes; ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - // printf("%s \n", int_nmr_field_name); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); @@ -315,7 +314,7 @@ void compute_tendencies( diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - for (int i=0; i::get_default_team_policy(ncol, nlev); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { + const int num_aero_modes = mam_coupling::num_aero_modes(); + MAMDryDep::view_2d dgncur_awet[num_aero_modes]; + MAMDryDep::view_2d wet_dens[num_aero_modes]; + + for (int i=0; i::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); @@ -437,19 +445,11 @@ void MAMDryDep::run_impl(const double dt) { // } // Kokkos::fence(); - view_2d dgncur_awet[num_aero_modes]; - for (int i=0; i Date: Wed, 6 Mar 2024 12:31:48 -0700 Subject: [PATCH 008/477] Fixes for Cuda build. --- .../physics/mam/eamxx_mam_dry_deposition_process_interface.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 03d44474bc86..aecea66a6c4a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -418,7 +418,6 @@ void MAMDryDep::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - const int num_aero_modes = mam_coupling::num_aero_modes(); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); From 05fe0fb235387002dd73464964f18e33aa3ec024 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 11 Mar 2024 13:40:17 -0600 Subject: [PATCH 009/477] Remove old comment as per code review. --- .../physics/mam/eamxx_mam_dry_deposition_process_interface.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index aecea66a6c4a..b12c5b12a1f3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -411,8 +411,7 @@ void fill_tracer_views( // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - // Include this but update when scream is updated to use the newer DryDeposition class: - using DryDep = mam4::DryDeposition; // mam4::DryDeposition; + using DryDep = mam4::DryDeposition; using MemberType = KT::MemberType; const auto scan_policy = ekat::ExeSpaceUtils< From 76ca0876a255864e5aa425144524aa62fed2d1ba Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 25 Mar 2024 12:08:12 -0600 Subject: [PATCH 010/477] Add p_int (pressure at interface) to DryAtmosphere structure. --- .../eamxx_mam_dry_deposition_process_interface.cpp | 6 +----- components/eamxx/src/physics/mam/mam_coupling.hpp | 11 +++++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index b12c5b12a1f3..86be78c461b5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -215,6 +215,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.p_del = get_field_in("pseudo_density").get_view(); dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.z_iface = buffer_.z_iface; dry_atm_.dz = buffer_.dz; @@ -421,8 +422,6 @@ void MAMDryDep::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - std::cout<<__FILE__<<":"<<__LINE__<<" In MAMDryDep::run_impl"< Date: Wed, 29 May 2024 17:03:52 -0700 Subject: [PATCH 011/477] work in progress mods to update definition of vtheta_dp to be consistent with v1 def in IOP interface --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 9e04b4a00920..2cd2ef95be66 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -18,6 +18,9 @@ #include "SimulationParams.hpp" #include "Types.hpp" +// SCREAM includes +#include "share/util/scream_common_physics_functions.hpp" + // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -227,6 +230,8 @@ apply_iop_forcing(const Real dt) using ElementOps = Homme::ElementOps; using KV = Homme::KernelVariables; + using PF = PhysicsFunctions; + using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -436,7 +441,7 @@ apply_iop_forcing(const Real dt) } // Update T and qv according to large scale forcing as specified in IOP file. - advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); +// advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); if (iop_coriolis) { // Apply coriolis forcing to u and v winds @@ -457,6 +462,13 @@ apply_iop_forcing(const Real dt) const int igp = idx/NGP; const int jgp = idx%NGP; + // Get temp views from workspace + auto ws = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); + + auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); @@ -466,6 +478,17 @@ apply_iop_forcing(const Real dt) auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); @@ -481,11 +504,13 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - // Recompute rstar with updated qv, and convert updated temperature back to potential temperature - elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); + // Convert updated temperature back to density weight virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + const auto th = PF::calculate_theta_from_T(temperature_i(k),s_pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); + // Release WS views + ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); }); From d04dc45f77cb514abe3890627c1918db3dd77ac7 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 30 May 2024 08:25:26 -0700 Subject: [PATCH 012/477] uncomment out advance_iop_forcing --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 2cd2ef95be66..cc3f2e6f3027 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -441,7 +441,7 @@ apply_iop_forcing(const Real dt) } // Update T and qv according to large scale forcing as specified in IOP file. -// advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); + advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); if (iop_coriolis) { // Apply coriolis forcing to u and v winds From 7a7333d268b432497137794ae6de99d8454e23c4 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sun, 2 Jun 2024 20:50:20 -0400 Subject: [PATCH 013/477] set explicit out and expand type --- components/eamxx/src/physics/p3/p3_iso_c.f90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index d75af1cb2edb..625ee48bc506 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,8 +16,8 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=256), intent(inout) :: string - character(*), intent(in) :: prefix + character(kind=c_char, len=256), intent(out) :: string + character(kind=c_char, len=*), intent(in) :: prefix real(kind=c_real) :: s write (string, '(a,i1,a1)') prefix, sizeof(s), C_NULL_CHAR From b2bd480441774296c4bc12803ae6b1d0dec1e48e Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sun, 2 Jun 2024 21:18:22 -0400 Subject: [PATCH 014/477] avoid specifying prefix type Co-authored-by: Andrew M. Bradley --- components/eamxx/src/physics/p3/p3_iso_c.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index 625ee48bc506..35c1154325f7 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -17,7 +17,7 @@ module p3_iso_c subroutine append_precision(string, prefix) character(kind=c_char, len=256), intent(out) :: string - character(kind=c_char, len=*), intent(in) :: prefix + character(*), intent(in) :: prefix real(kind=c_real) :: s write (string, '(a,i1,a1)') prefix, sizeof(s), C_NULL_CHAR From d0a7290ff1bf7f8f4fc09bfe2125ce927d7909ba Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 3 Jun 2024 14:47:24 -0400 Subject: [PATCH 015/477] use char concat instead of write routine --- components/eamxx/src/physics/p3/p3_iso_c.f90 | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index 35c1154325f7..5f887607c182 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=256), intent(out) :: string + character(kind=c_char, len=512), intent(out) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,10 +69,17 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. - call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") - call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") - call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") - call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") +#ifdef SCREAM_DOUBLE_PRECISION + mu_r_filename = lookup_file_dir//'/mu_r_table_vals.dat8'//C_NULL_CHAR + revap_filename = lookup_file_dir//'/revap_table_vals.dat8'//C_NULL_CHAR + vn_filename = lookup_file_dir//'/vn_table_vals.dat8'//C_NULL_CHAR + vm_filename = lookup_file_dir//'/vm_table_vals.dat8'//C_NULL_CHAR +#else + mu_r_filename = lookup_file_dir//'/mu_r_table_vals.dat4'//C_NULL_CHAR + revap_filename = lookup_file_dir//'/revap_table_vals.dat4'//C_NULL_CHAR + vn_filename = lookup_file_dir//'/vn_table_vals.dat4'//C_NULL_CHAR + vm_filename = lookup_file_dir//'/vm_table_vals.dat4'//C_NULL_CHAR +#endif if (write_tables) then call p3_init_b() From d2ed08cf79276ae8e9fc9bab3bb16bc5aa0eb53d Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 3 Jun 2024 15:21:27 -0400 Subject: [PATCH 016/477] null char offset --- components/eamxx/src/physics/p3/p3_iso_c.f90 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index 5f887607c182..ea0a18411c10 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -70,15 +70,15 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) ok = .false. #ifdef SCREAM_DOUBLE_PRECISION - mu_r_filename = lookup_file_dir//'/mu_r_table_vals.dat8'//C_NULL_CHAR - revap_filename = lookup_file_dir//'/revap_table_vals.dat8'//C_NULL_CHAR - vn_filename = lookup_file_dir//'/vn_table_vals.dat8'//C_NULL_CHAR - vm_filename = lookup_file_dir//'/vm_table_vals.dat8'//C_NULL_CHAR + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR #else - mu_r_filename = lookup_file_dir//'/mu_r_table_vals.dat4'//C_NULL_CHAR - revap_filename = lookup_file_dir//'/revap_table_vals.dat4'//C_NULL_CHAR - vn_filename = lookup_file_dir//'/vn_table_vals.dat4'//C_NULL_CHAR - vm_filename = lookup_file_dir//'/vm_table_vals.dat4'//C_NULL_CHAR + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR #endif if (write_tables) then From bfc54464aa92a62159127aae5cab9c4a3599fb0f Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 4 Jun 2024 10:12:06 -0700 Subject: [PATCH 017/477] update temperature consistent with defintions of vtheta_dp that homme uses, two way --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index cc3f2e6f3027..6c7f41cf5972 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -355,10 +355,17 @@ apply_iop_forcing(const Real dt) auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); + // Get temp views from workspace + auto ws2 = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws2.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; const int jgp = idx%NGP; + auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); @@ -368,6 +375,18 @@ apply_iop_forcing(const Real dt) auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + team.team_barrier(); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); @@ -389,12 +408,19 @@ apply_iop_forcing(const Real dt) eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); } - // Get the temperature from dynamics states - elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + // Compute temperature + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + auto& T_val = vtheta_dp_i(k); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); + }); + }); // Release WS views ws.release_macro_block(pnh_slot, NGP*NGP); + ws2.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); }; @@ -489,6 +515,8 @@ apply_iop_forcing(const Real dt) } }); + team.team_barrier(); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); @@ -504,11 +532,12 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - // Convert updated temperature back to density weight virtual potential temperature + // Recompute rstar with updated qv, and convert updated temperature back to potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),s_pmid(k)); + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); + // Release WS views ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); From 0161bd7a45a1c879794a1995a5b571f867685c04 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 5 Jun 2024 17:08:33 -0600 Subject: [PATCH 018/477] Port scream_rrtmgp_interface to new pattern --- .../eamxx/src/physics/rrtmgp/CMakeLists.txt | 1 - .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 6 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 1167 +------------- .../rrtmgp/scream_rrtmgp_interface.hpp | 1432 ++++++++++++++--- 4 files changed, 1243 insertions(+), 1363 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index c098a0829d52..c24c9d9fd1aa 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -109,7 +109,6 @@ set(EAM_RRTMGP_DIR ${SCREAM_BASE_DIR}/../eam/src/physics/rrtmgp) # NOTE: The external RRTMGP build needs some fixes to work with CUDA in a library build, so for now we will build these ourselves set(EXTERNAL_SRC ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels/mo_gas_optics_kernels.cpp - ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_constants.cpp ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_util_reorder.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/expand_and_transpose.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_fluxes_broadband_kernels.cpp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 0dfe4945e0c8..119093d5f9d3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -109,14 +109,14 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, RealT xmin, RealT xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - bool1dk bad_mask("bad_mask", x.size()); + Kokkos::View bad_mask("bad_mask", x.size()); Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { if (x.data()[i] < xmin or x.data()[i] > xmax) { bad_mask.data()[i] = true; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 450d7ad216be..1696e7933685 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,5 +1,4 @@ #include "scream_rrtmgp_interface.hpp" -#include "rrtmgp_utils.hpp" #include "cpp/examples/mo_load_coefficients.h" #include "examples/all-sky/mo_load_cloud_coefficients.h" #include "cpp/rrtmgp/mo_gas_concentrations.h" @@ -8,10 +7,6 @@ #include "cpp/rte/mo_rte_sw.h" #include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" -#include "ekat/util/ekat_math_utils.hpp" -#ifdef RRTMGP_ENABLE_KOKKOS -#include "Kokkos_Random.hpp" -#endif namespace scream { @@ -38,41 +33,27 @@ void finalize_kls() #endif } +#ifdef RRTMGP_ENABLE_YAKL namespace rrtmgp { -#ifdef RRTMGP_ENABLE_YAKL using yakl::fortran::parallel_for; using yakl::fortran::SimpleBounds; using yakl::intrinsics::merge; -#endif - /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -#ifdef RRTMGP_ENABLE_YAKL GasOpticsRRTMGP k_dist_sw; GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -GasOpticsRRTMGPK k_dist_sw_k; -GasOpticsRRTMGPK k_dist_lw_k; -#endif /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -#ifdef RRTMGP_ENABLE_YAKL CloudOptics cloud_optics_sw; CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -CloudOpticsK cloud_optics_sw_k; -CloudOpticsK cloud_optics_lw_k; -#endif bool initialized = false; bool initialized_k = false; @@ -80,7 +61,6 @@ bool initialized_k = false; // local functions namespace { -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_cloud_optics_sw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -106,36 +86,7 @@ OpticalProps2str get_cloud_optics_sw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_cloud_optics_sw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps2strK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_2str(ncol, nlay); - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_cloud_optics_lw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -161,36 +112,7 @@ OpticalProps1scl get_cloud_optics_lw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_cloud_optics_lw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps1sclK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2str &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -242,60 +164,7 @@ OpticalProps2str get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps2strK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_2str(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, - // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), - // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should - // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very - // high resolution. - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); - subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - subsampled_optics.ssa(icol,ilay,igpt) = 0; - subsampled_optics.g (icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1scl &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -339,51 +208,6 @@ OpticalProps1scl get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_1scl(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM; use different - // seed values for longwave and shortwave - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif } @@ -392,7 +216,6 @@ OpticalProps1sclK get_subsampled_clouds( * can be used as-is, but are intended to be wrapped by the SCREAM AD * interface to radiation. */ -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_initialize(GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, @@ -419,65 +242,15 @@ void rrtmgp_initialize(GasConcs &gas_concs, // We are now initialized! initialized = true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger) -{ - // If we've already initialized, just exit - if (initialized_k) { - if (logger) - logger->info("RRTMGP is already initialized; skipping\n"); - return; - } - - // Initialize Kokkos - if (!Kokkos::is_initialized()) { Kokkos::initialize(); } - - // Load and initialize absorption coefficient data - load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); - load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); - - // Load and initialize cloud optical property look-up table information - load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); - load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); - - // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 18000; - const size_t ncol = gas_concs.ncol; - const size_t nlay = gas_concs.nlay; - const size_t nlev = SCREAM_NUM_VERTICAL_LEV; - const size_t my_size_ref = ncol * nlay * nlev; - conv::MemPoolSingleton::init(2e6 * (float(my_size_ref) / base_ref)); - - // We are now initialized! - initialized_k = true; -} -#endif - void rrtmgp_finalize() { -#ifdef RRTMGP_ENABLE_YAKL initialized = false; k_dist_sw.finalize(); k_dist_lw.finalize(); cloud_optics_sw.finalize(); //~CloudOptics(); cloud_optics_lw.finalize(); //~CloudOptics(); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - initialized_k = false; - k_dist_sw_k.finalize(); - k_dist_lw_k.finalize(); - cloud_optics_sw_k.finalize(); //~CloudOptics(); - cloud_optics_lw_k.finalize(); //~CloudOptics(); - conv::MemPoolSingleton::finalize(); -#endif } -#ifdef RRTMGP_ENABLE_YAKL void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, @@ -527,56 +300,7 @@ void compute_band_by_band_surface_albedos( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif) { - - EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - - EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, - "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); - EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), - "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); - - // Loop over bands, and determine for each band whether it is broadly in the - // visible or infrared part of the spectrum (visible or "not visible") - Kokkos::parallel_for(conv::get_mdrp<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); - } - else { - // Band straddles the visible to near-infrared transition, so we take - // the albedo to be the average of the visible and near-infrared - // broadband albedos - sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); - sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , @@ -631,62 +355,7 @@ void compute_broadband_surface_fluxes( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir) { - // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums - // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when - // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This - // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. - //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - - // Initialize sums over bands - Kokkos::deep_copy(sfc_flux_dir_nir, 0); - Kokkos::deep_copy(sfc_flux_dir_vis, 0); - Kokkos::deep_copy(sfc_flux_dif_nir, 0); - Kokkos::deep_copy(sfc_flux_dif_vis, 0); - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { - for (int ibnd = 0; ibnd < nswbands; ++ibnd) { - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else { - // Band straddles the visible to near-infrared transition, so put half - // the flux in visible and half in near-infrared fluxes - sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - } - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -855,191 +524,19 @@ void rrtmgp_main( ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, - real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - -#ifdef SCREAM_RRTMGP_DEBUG - // Sanity check inputs, and possibly repair - check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); - check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); - check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); - check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); - check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); - check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); - check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); - check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); - check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); - check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); - check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); -#endif - - // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; - fluxes_sw.flux_up = sw_flux_up; - fluxes_sw.flux_dn = sw_flux_dn; - fluxes_sw.flux_dn_dir = sw_flux_dn_dir; - fluxes_sw.bnd_flux_up = sw_bnd_flux_up; - fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; - fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; - clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; - clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; - clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; - clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; - clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; - clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; - clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; - clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; - clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; - // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; - fluxes_lw.flux_up = lw_flux_up; - fluxes_lw.flux_dn = lw_flux_dn; - fluxes_lw.bnd_flux_up = lw_bnd_flux_up; - fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; - clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; - clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; - clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; - clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; - clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; - clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; +int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - auto nswbands = k_dist_sw_k.get_nband(); - auto nlwbands = k_dist_lw_k.get_nband(); + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; - aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); - aerosol_sw.alloc_2str(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); - aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); - aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); - }); - aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); - aerosol_lw.alloc_1scl(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Check aerosol optical properties - // NOTE: these should already have been checked by precondition checks, but someday we might have - // non-trivial aerosol optics, so this is still good to do here. - check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); - check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); - check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); - check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); -#endif - - // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); - Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); - Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); - - // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; - // This implements the Monte Carlo Independing Column Approximation by mapping only a single - // subcolumn (cloud state) to each gpoint. - auto nswgpts = k_dist_sw_k.get_ngpt(); - auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); - // Longwave - auto nlwgpts = k_dist_lw_k.get_ngpt(); - auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); - - // Copy cloud properties to outputs (is this needed, or can we just use pointers?) - // Alternatively, just compute and output a subcolumn cloud mask - Kokkos::parallel_for(conv::get_mdrp<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, - // but we might want to provide additional debug info here. NOTE: we may actually want to move this - // up higher in the code, I think optical props should go up higher since optical props are kind of - // a parameterization of their own, and we might want to swap different choices. These checks go here - // only because we need to run them on computed optical props, so if the optical props themselves get - // computed up higher, then perform these checks higher as well - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); - check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); - check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); -#endif - - // Do shortwave - rrtmgp_sw( - ncol, nlay, - k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, - fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, - tsi_scaling, logger, - extra_clnclrsky_diag, extra_clnsky_diag - ); - - // Do longwave - rrtmgp_lw( - ncol, nlay, - k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - aerosol_lw, clouds_lw_gpt, - fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, - extra_clnclrsky_diag, extra_clnsky_diag - ); - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL -int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3d("cldx", ncol, nlay, ngpt); + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = real3d("cldx", ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1090,84 +587,7 @@ int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d }); return subcolumn_mask; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3dk("cldx", ncol, nlay, ngpt); - - // Apply overlap assumption to set cldx - if (overlap_option == 0) { // Dummy mask, always cloudy - Kokkos::deep_copy(cldx, 1); - } else { // Default case, maximum-random overlap - // Maximum-random overlap: - // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, - // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same - // algorithm used in RRTMG implementation of maximum-random overlap (see - // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) - // - // First, fill cldx with random numbers. Need to use a unique seed for each column! - // auto seeds_host = Kokkos::create_mirror_view(seeds); - // Kokkos::deep_copy(seeds_host, seeds); - // for (int icol = 0; icol < ncol; ++icol) { - // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { - // auto generator = random_pool.get_state(); - // cldx(icol,ilay,igpt) = generator.drand(0., 1.); - // random_pool.free_state(generator); - // }); - // } - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - conv::Random rand(seeds(icol)); - for (int igpt = 0; igpt < ngpt; igpt++) { - for (int ilay = 0; ilay < nlay; ilay++) { - cldx(icol,ilay,igpt) = rand.genFP(); - } - } - }); - - // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { - for (int ilay = 1; ilay < nlay; ilay++) { - // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn - if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { - // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent - // layers are maximimally overlapped - cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); - } else { - // Cloud-less above, use new random number so that clouds are distributed - // randomly in this layer. Need to scale new random number to range - // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution - // of random numbers in this layer with the above branch of the conditional, - // which would otherwise inflate cloud fraction in this layer. - cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); - } - } - }); - } - - // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { - subcolumn_mask(icol,ilay,igpt) = 1; - } else { - subcolumn_mask(icol,ilay,igpt) = 0; - } - }); - return subcolumn_mask; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1423,267 +843,7 @@ void rrtmgp_sw( }); } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &flux_dn_dir = fluxes.flux_dn_dir; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; - - // Reset fluxes to zero - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up (icol,ilev) = 0; - flux_dn (icol,ilev) = 0; - flux_dn_dir(icol,ilev) = 0; - clnclrsky_flux_up (icol,ilev) = 0; - clnclrsky_flux_dn (icol,ilev) = 0; - clnclrsky_flux_dn_dir(icol,ilev) = 0; - clrsky_flux_up (icol,ilev) = 0; - clrsky_flux_dn (icol,ilev) = 0; - clrsky_flux_dn_dir(icol,ilev) = 0; - clnsky_flux_up (icol,ilev) = 0; - clnsky_flux_dn (icol,ilev) = 0; - clnsky_flux_dn_dir(icol,ilev) = 0; - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up (icol,ilev,ibnd) = 0; - bnd_flux_dn (icol,ilev,ibnd) = 0; - bnd_flux_dn_dir(icol,ilev,ibnd) = 0; - }); - - // Get daytime indices - auto dayIndices = int1dk("dayIndices", ncol); - Kokkos::deep_copy(dayIndices, -1); - - int nday = 0; - // Serialized for now. - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { - for (int icol = 0; icol < ncol; ++icol) { - if (mu0(icol) > 0) { - dayIndices(nday_inner++) = icol; - } - } - }, Kokkos::Sum(nday)); - - if (nday == 0) { - // No daytime columns in this chunk, skip the rest of this routine - return; - } - - // Subset mu0 - auto mu0_day = real1dk("mu0_day", nday); - Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { - mu0_day(iday) = mu0(dayIndices(iday)); - }); - - // subset state variables - auto p_lay_day = real2dk("p_lay_day", nday, nlay); - auto t_lay_day = real2dk("t_lay_day", nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); - t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); - }); - auto p_lev_day = real2dk("p_lev_day", nday, nlay+1); - auto t_lev_day = real2dk("t_lev_day", nday, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); - t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); - }); - - // Subset gases - auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; - gas_concs_day.init(gas_names, nday, nlay); - for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = real2dk("vmr_day", nday, nlay); - auto vmr = real2dk("vmr" , ncol, nlay); - gas_concs.get_vmr(gas_names[igas], vmr); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); - }); - gas_concs_day.set_vmr(gas_names[igas], vmr_day); - } - - // Subset aerosol optics - OpticalProps2strK aerosol_day; - aerosol_day.init(k_dist.get_band_lims_wavenumber()); - aerosol_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { - aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); - aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); - aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); - }); - - // Subset cloud optics - // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; - clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); - clouds_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { - clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); - clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); - clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); - }); - - // RRTMGP assumes surface albedos have a screwy dimension ordering - // for some strange reason, so we need to transpose these; also do - // daytime subsetting in the same kernel - real2dk sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - real2dk sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); - Kokkos::parallel_for(conv::get_mdrp<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { - sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); - sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); - }); - - // Temporaries we need for daytime-only fluxes - auto flux_up_day = real2dk("flux_up_day", nday, nlay+1); - auto flux_dn_day = real2dk("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = real2dk("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = real3dk("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = real3dk("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = real3dk("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; - fluxes_day.flux_up = flux_up_day; - fluxes_day.flux_dn = flux_dn_day; - fluxes_day.flux_dn_dir = flux_dn_dir_day; - fluxes_day.bnd_flux_up = bnd_flux_up_day; - fluxes_day.bnd_flux_dn = bnd_flux_dn_day; - fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; - - // Allocate space for optical properties - OpticalProps2strK optics; - optics.alloc_2str(nday, nlay, k_dist); - - OpticalProps2strK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_2str(nday, nlay, k_dist); - } - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", nday, nlay); - limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); - - // Do gas optics - real2dk toa_flux("toa_flux", nday, ngpt); - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); - if (extra_clnsky_diag) { - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); - check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); - check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); -#endif - - // Apply tsi_scaling - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { - toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); - }); - - if (extra_clnclrsky_diag) { - // Compute clear-clean-sky (just gas) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } - - // Combine gas and aerosol optics - aerosol_day.delta_scale(); - aerosol_day.increment(optics); - - // Compute clearsky (gas + aerosol) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - - // Now merge in cloud optics and do allsky calculations - - // Combine gas and cloud optics - clouds_day.delta_scale(); - clouds_day.increment(optics); - // Compute fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - flux_up (icol,ilev) = flux_up_day (iday,ilev); - flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { - const int icol = dayIndices(iday); - bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); - bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); - bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); - }); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds_day.increment(optics_no_aerosols); - // Compute cleansky (gas + clouds) fluxes on daytime columns - rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1816,147 +976,7 @@ void rrtmgp_lw( } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, - OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Problem size - int nbnd = k_dist.get_nband(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - - // Reset fluxes to zero - Kokkos::parallel_for( - conv::get_mdrp<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up(icol, ilev) = 0; - flux_dn(icol, ilev) = 0; - clnclrsky_flux_up(icol, ilev) = 0; - clnclrsky_flux_dn(icol, ilev) = 0; - clrsky_flux_up(icol, ilev) = 0; - clrsky_flux_dn(icol, ilev) = 0; - clnsky_flux_up(icol, ilev) = 0; - clnsky_flux_dn(icol, ilev) = 0; - }); - Kokkos::parallel_for( - conv::get_mdrp<3>({nbnd, nlay + 1, ncol}), - KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up(icol, ilev, ibnd) = 0; - bnd_flux_dn(icol, ilev, ibnd) = 0; - }); - // Allocate space for optical properties - OpticalProps1sclK optics; - optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); - } - - // Boundary conditions - SourceFuncLWK lw_sources; - lw_sources.alloc(ncol, nlay, k_dist); - real1dk t_sfc ("t_sfc" ,ncol); - real2dk emis_sfc("emis_sfc",nbnd,ncol); - - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - // Surface temperature - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); - }); - Kokkos::deep_copy(emis_sfc , 0.98); - - // Get Gaussian quadrature weights - // TODO: move this crap out of userland! - // Weights and angle secants for first order (k=1) Gaussian quadrature. - // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 - // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - realHost2dk gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - realHost2dk gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - real2dk gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - real2dk gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); - Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); - Kokkos::deep_copy(gauss_wts, gauss_wts_host); - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", ncol, nlay); - auto t_lev_limited = real2dk("t_lev_limited", ncol, nlay+1); - limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); - limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); - - // Do gas optics - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, real2dk(), t_lev_limited); - if (extra_clnsky_diag) { - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, real2dk(), t_lev_limited); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); -#endif - - if (extra_clnclrsky_diag) { - // Compute clean-clear-sky fluxes before we add in aerosols and clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); - } - - // Combine gas and aerosol optics - aerosol.increment(optics); - - // Compute clear-sky fluxes before we add in clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); - - // Combine gas and cloud optics - clouds.increment(optics); - - // Compute allsky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds.increment(optics_no_aerosols); - // Compute clean-sky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); - } - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area) { @@ -1981,46 +1001,11 @@ void compute_cloud_area( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area) { - // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy - // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = real2dk("subcol_mask", ncol, ngpt); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but - // using play/pmid does not - if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { - subcol_mask(icol,igpt) = 1; - } - }); - // Compute average over subcols to get cloud area - auto ngpt_inv = 1.0 / ngpt; - Kokkos::deep_copy(cld_area, 0); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // This loop needs to be serial because of the atomic reduction - for (int igpt = 0; igpt < ngpt; ++igpt) { - cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index_sw_k(double wavelength) { return get_wavelength_index(k_dist_sw_k, wavelength); } - -int get_wavelength_index_lw_k(double wavelength) { return get_wavelength_index(k_dist_lw_k, wavelength); } -#endif - -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index(OpticalProps &kdist, double wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -2043,33 +1028,7 @@ int get_wavelength_index(OpticalProps &kdist, double wavelength) { }); return band_index.hostRead(); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength) { - // Get wavelength bounds for all wavelength bands - auto wavelength_bounds = kdist.get_band_lims_wavelength(); - - // Find the band index for the specified wavelength - // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength - // in units of meters, we need a conversion factor of 10^2 - const int nbnds = kdist.get_nband(); - int band_index = -1; - Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { - if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { - if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } else { - if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } - }, Kokkos::Max(band_index)); - return band_index; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -2166,109 +1125,7 @@ void compute_aerocom_cloudtop( cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop) { - /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCom recommendation. See reference for routine - * get_subcolumn_mask above, where equation 14 is used for the - * maximum-random overlap assumption for subcolumn generation. We use - * equation 13, the column counterpart. - */ - // Set outputs to zero - Kokkos::deep_copy(T_mid_at_cldtop, 0.0); - Kokkos::deep_copy(p_mid_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); - Kokkos::deep_copy(cdnc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); - - // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = real1dk("aerocom_clr", ncol); - Kokkos::deep_copy(aerocom_clr, 1.0); - - // Get gravity acceleration constant from constants - using physconst = scream::physics::Constants; - - // TODO: move tunable constant to namelist - constexpr real q_threshold = 0.0; // BAD_CONSTANT! - - // TODO: move tunable constant to namelist - constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! - - // Loop over all columns in parallel - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // Loop over all layers in serial (due to accumulative - // product), starting at 2 (second highest) layer because the - // highest is assumed to hav no clouds - for(int ilay = 1; ilay < nlay; ++ilay) { - // Only do the calculation if certain conditions are met - if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && - (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { - /* PART I: Probabilistically determining cloud top */ - // Populate aerocom_tmp as the clear-sky fraction - // probability of this level, where aerocom_clr is that of - // the previous level - auto aerocom_tmp = - aerocom_clr(icol) * - (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), - cldfrac_tot(icol, ilay))) / - (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), - 1.0 - cldfrac_tot_threshold)); - // Temporary variable for probability "weights" - auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; - // Temporary variable for liquid "phase" - auto aerocom_phi = - qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); - /* PART II: The inferred properties */ - /* In general, converting a 3D property X to a 2D cloud-top - * counterpart x follows: x(i) += X(i,k) * weights * Phase - * but X and Phase are not always needed */ - // T_mid_at_cldtop - T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; - // p_mid_at_cldtop - p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; - // cldfrac_ice_at_cldtop - cldfrac_ice_at_cldtop(icol) += - (1.0 - aerocom_phi) * aerocom_wts; - // cldfrac_liq_at_cldtop - cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; - // cdnc_at_cldtop - /* We need to convert nc from 1/mass to 1/volume first, and - * from grid-mean to in-cloud, but after that, the - * calculation follows the general logic */ - auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / - z_del(icol, ilay) / physconst::gravit / - cldfrac_tot(icol, ilay); - cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; - // eff_radius_qc_at_cldtop - eff_radius_qc_at_cldtop(icol) += - rel(icol, ilay) * aerocom_phi * aerocom_wts; - // eff_radius_qi_at_cldtop - eff_radius_qi_at_cldtop(icol) += - rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset aerocom_clr to aerocom_tmp to accumulate - aerocom_clr(icol) = aerocom_tmp; - } - } - // After the serial loop over levels, the cloudy fraction is - // defined as (1 - aerocom_clr). This is true because - // aerocom_clr is the result of accumulative probabilities - // (their products) - cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); - }); -} -#endif } // namespace rrtmgp +#endif } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 49fbfeb4486b..c40be4904302 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -6,10 +6,17 @@ #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" #include "cpp/rrtmgp_const.h" +#include "rrtmgp_utils.hpp" + #include "physics/share/physics_constants.hpp" #include "ekat/mpi/ekat_comm.hpp" #include "ekat/logging/ekat_logger.hpp" +#include "ekat/util/ekat_math_utils.hpp" + +#ifdef RRTMGP_ENABLE_KOKKOS +#include "Kokkos_Random.hpp" +#endif namespace scream { @@ -18,105 +25,34 @@ void finalize_kls(); namespace rrtmgp { -/* - * Objects containing k-distribution information need to be initialized - * once and then persist throughout the life of the program, so we - * declare them here within the rrtmgp namespace. - */ #ifdef RRTMGP_ENABLE_YAKL extern GasOpticsRRTMGP k_dist_sw; extern GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern GasOpticsRRTMGPK k_dist_sw_k; -extern GasOpticsRRTMGPK k_dist_lw_k; -#endif -/* - * Objects containing cloud optical property look-up table information. - * We want to initialize these once and use throughout the life of the - * program, so declare here and read data in during rrtmgp_initialize(). - */ -#ifdef RRTMGP_ENABLE_YAKL extern CloudOptics cloud_optics_sw; extern CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern CloudOpticsK cloud_optics_sw_k; -extern CloudOpticsK cloud_optics_lw_k; -#endif -/* - * Flag to indicate whether or not we have initialized RRTMGP - */ -#ifdef RRTMGP_ENABLE_YAKL extern bool initialized; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern bool initialized_k; -#endif -/* - * Initialize data for RRTMGP driver - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_initialize( +void rrtmgp_initialize( GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger); -#endif -/* - * Compute band-by-band surface albedos from broadband albedos. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_band_by_band_surface_albedos( +void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, real2d &sfc_alb_dir, real2d &sfc_alb_dif); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif); -#endif -/* - * Compute broadband visible/UV and near-infrared surface fluxes. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_broadband_surface_fluxes( +void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir); -#endif -/* - * Main driver code to run RRTMGP. - * The input logger is in charge of outputing info to - * screen and/or to file (or neither), depending on how it was set up. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_main( +void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, GasConcs &gas_concs, @@ -138,42 +74,10 @@ extern void rrtmgp_main( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -/* - * Perform any clean-up tasks - */ -extern void rrtmgp_finalize(); +void rrtmgp_finalize(); -/* - * Shortwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_sw( +void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -184,26 +88,8 @@ extern void rrtmgp_sw( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Longwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_lw( +void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -211,46 +97,13 @@ extern void rrtmgp_lw( OpticalProps1scl &aerosol, OpticalProps1scl &clouds, FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Return a subcolumn mask consistent with a specified overlap assumption - */ -#ifdef RRTMGP_ENABLE_YAKL int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds); -#endif -/* - * Compute cloud area from 3d subcol cloud property - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, Real pmin, Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, Real pmin, Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area); -#endif -/* - * Return select cloud-top diagnostics following AeroCom recommendation - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -260,26 +113,7 @@ void compute_aerocom_cloudtop( real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop); -#endif -/* - * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area - * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion - * with the standard practice of using "water path" to refer to the total column-integrated - * quantities). - */ -#ifdef RRTMGP_ENABLE_YAKL template void mixing_ratio_to_cloud_mass( yakl::Array const &mixing_ratio, @@ -303,10 +137,1035 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif + +template +void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { + yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { + arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + }); +} + +int get_wavelength_index(OpticalProps &kdist, double wavelength); +int get_wavelength_index_sw(double wavelength); +int get_wavelength_index_lw(double wavelength); +#endif // RRTMGP_ENABLE_YAKL + +// New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS +template +struct rrtmgp_interface { + +using MDRP = typename conv::MDRP; + +template +using view_t = Kokkos::View; + +template +using oview_t = Kokkos::Experimental::OffsetView; + +template +using hview_t = Kokkos::View; + +using pool_t = conv::MemPoolSingleton; + +/* + * Objects containing k-distribution information need to be initialized + * once and then persist throughout the life of the program, so we + * declare them here within the rrtmgp namespace. + */ +static inline GasOpticsRRTMGPK k_dist_sw_k; +static inline GasOpticsRRTMGPK k_dist_lw_k; + +/* + * Objects containing cloud optical property look-up table information. + * We want to initialize these once and use throughout the life of the + * program, so declare here and read data in during rrtmgp_initialize(). + */ +static inline CloudOpticsK cloud_optics_sw_k; +static inline CloudOpticsK cloud_optics_lw_k; + +/* + * Flag to indicate whether or not we have initialized RRTMGP + */ +static inline bool initialized_k = false; + +/* + * Initialize data for RRTMGP driver + */ +static void rrtmgp_initialize( + GasConcsK &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger) +{ + // If we've already initialized, just exit + if (initialized_k) { + if (logger) + logger->info("RRTMGP is already initialized; skipping\n"); + return; + } + + // Initialize Kokkos + if (!Kokkos::is_initialized()) { Kokkos::initialize(); } + + // Load and initialize absorption coefficient data + load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); + load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); + + // Load and initialize cloud optical property look-up table information + load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); + load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); + + // initialize kokkos rrtmgp pool allocator + const size_t base_ref = 18000; + const size_t ncol = gas_concs.ncol; + const size_t nlay = gas_concs.nlay; + const size_t nlev = SCREAM_NUM_VERTICAL_LEV; + const size_t my_size_ref = ncol * nlay * nlev; + pool_t::init(2e6 * (float(my_size_ref) / base_ref)); + + // We are now initialized! + initialized_k = true; +} + +/* + * Compute band-by-band surface albedos from broadband albedos. + */ +template +static void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + SfcAlbDirVisT &sfc_alb_dir_vis, SfcAlbDirNirT &sfc_alb_dir_nir, + SfcAlbDifVisT &sfc_alb_dif_vis, SfcAlbDifNirT &sfc_alb_dif_nir, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif) +{ + EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + + EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, + "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); + EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), + "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); + + // Loop over bands, and determine for each band whether it is broadly in the + // visible or infrared part of the spectrum (visible or "not visible") + Kokkos::parallel_for(MDRP::template get<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); + } + else { + // Band straddles the visible to near-infrared transition, so we take + // the albedo to be the average of the visible and near-infrared + // broadband albedos + sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); + sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); + } + }); +} + +/* + * Compute broadband visible/UV and near-infrared surface fluxes. + */ +template +static void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + SwBndFluxDirT &sw_bnd_flux_dir , SwBndFluxDifT &sw_bnd_flux_dif , + SfcFluxDirVisT &sfc_flux_dir_vis, SfcFluxDirNirT &sfc_flux_dir_nir, + SfcFluxDifVisT &sfc_flux_dif_vis, SfcFluxDifNirT &sfc_flux_dif_nir) +{ + // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums + // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when + // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This + // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. + //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + + // Initialize sums over bands + Kokkos::deep_copy(sfc_flux_dir_nir, 0); + Kokkos::deep_copy(sfc_flux_dir_vis, 0); + Kokkos::deep_copy(sfc_flux_dif_nir, 0); + Kokkos::deep_copy(sfc_flux_dif_vis, 0); + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { + for (int ibnd = 0; ibnd < nswbands; ++ibnd) { + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else { + // Band straddles the visible to near-infrared transition, so put half + // the flux in visible and half in near-infrared fluxes + sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + } + } + }); +} + +/* + * Main driver code to run RRTMGP. + * The input logger is in charge of outputing info to + * screen and/or to file (or neither), depending on how it was set up. + */ +template +static void rrtmgp_main( + const int ncol, const int nlay, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei, CldfracT &cldfrac, + AerTauSwT &aer_tau_sw, AerSsaSwT &aer_ssa_sw, AerAsmSwT &aer_asm_sw, AerTauLwT &aer_tau_lw, + CldTauSwBndT &cld_tau_sw_bnd, CldTauLwBndT &cld_tau_lw_bnd, + CldTauSwGptT &cld_tau_sw_gpt, CldTauLwGptT &cld_tau_lw_gpt, + SwFluxUpT &sw_flux_up, SwFluxDnT &sw_flux_dn, SwFluxDnDirT &sw_flux_dn_dir, + LwFluxUpT &lw_flux_up, LwFluxDnT &lw_flux_dn, + SwClnclrskyFluxUpT &sw_clnclrsky_flux_up, SwClnclrskyFluxDnT &sw_clnclrsky_flux_dn, SwClnclrskyFluxDnDirT &sw_clnclrsky_flux_dn_dir, + SwClrskyFluxUpT &sw_clrsky_flux_up, SwClrskyFluxDnT &sw_clrsky_flux_dn, SwClrskyFluxDnDirT &sw_clrsky_flux_dn_dir, + SwClnskyFluxUpT &sw_clnsky_flux_up, SwClnskyFluxDnT &sw_clnsky_flux_dn, SwClnskyFluxDnDirT &sw_clnsky_flux_dn_dir, + LwClnclrskyFluxUpT &lw_clnclrsky_flux_up, LwClnclrskyFluxDnT &lw_clnclrsky_flux_dn, + LwClrskyFluxUpT &lw_clrsky_flux_up, LwClrskyFluxDnT &lw_clrsky_flux_dn, + LwClnskyFluxUpT &lw_clnsky_flux_up, LwClnskyFluxDnT &lw_clnsky_flux_dn, + SwBndFluxUpT &sw_bnd_flux_up, SwBndFluxDnT &sw_bnd_flux_dn, SwBndFluxDnDirT &sw_bnd_flux_dn_dir, + LwBndFluxUpT &lw_bnd_flux_up, LwBndFluxDnT &lw_bnd_flux_dn, + const RealT tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) +{ +#ifdef SCREAM_RRTMGP_DEBUG + // Sanity check inputs, and possibly repair + check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); + check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); + check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); + check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); + check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); + check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); + check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); + check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); + check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); + check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); + check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); +#endif + + // Setup pointers to RRTMGP SW fluxes + FluxesBybandK fluxes_sw; + fluxes_sw.flux_up = sw_flux_up; + fluxes_sw.flux_dn = sw_flux_dn; + fluxes_sw.flux_dn_dir = sw_flux_dn_dir; + fluxes_sw.bnd_flux_up = sw_bnd_flux_up; + fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; + fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_sw; + clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; + clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; + clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_sw; + clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; + clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; + clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_sw; + clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; + clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; + clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; + + // Setup pointers to RRTMGP LW fluxes + FluxesBybandK fluxes_lw; + fluxes_lw.flux_up = lw_flux_up; + fluxes_lw.flux_dn = lw_flux_dn; + fluxes_lw.bnd_flux_up = lw_bnd_flux_up; + fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_lw; + clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; + clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_lw; + clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; + clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_lw; + clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; + clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; + + auto nswbands = k_dist_sw_k.get_nband(); + auto nlwbands = k_dist_lw_k.get_nband(); + + // Setup aerosol optical properties + OpticalProps2strK aerosol_sw; + OpticalProps1sclK aerosol_lw; + aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); + aerosol_sw.alloc_2str(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); + aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); + aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); + }); + aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); + aerosol_lw.alloc_1scl(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Check aerosol optical properties + // NOTE: these should already have been checked by precondition checks, but someday we might have + // non-trivial aerosol optics, so this is still good to do here. + check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); + check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); + check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); + check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); +#endif + + // Convert cloud physical properties to optical properties for input to RRTMGP + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); + Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); + + // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; + // This implements the Monte Carlo Independing Column Approximation by mapping only a single + // subcolumn (cloud state) to each gpoint. + auto nswgpts = k_dist_sw_k.get_ngpt(); + auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); + // Longwave + auto nlwgpts = k_dist_lw_k.get_ngpt(); + auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); + + // Copy cloud properties to outputs (is this needed, or can we just use pointers?) + // Alternatively, just compute and output a subcolumn cloud mask + Kokkos::parallel_for(MDRP::template get<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); + }); + Kokkos::parallel_for(MDRP::template get<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, + // but we might want to provide additional debug info here. NOTE: we may actually want to move this + // up higher in the code, I think optical props should go up higher since optical props are kind of + // a parameterization of their own, and we might want to swap different choices. These checks go here + // only because we need to run them on computed optical props, so if the optical props themselves get + // computed up higher, then perform these checks higher as well + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); + check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); +#endif + + // Do shortwave + rrtmgp_sw( + ncol, nlay, + k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, + fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, + tsi_scaling, logger, + extra_clnclrsky_diag, extra_clnsky_diag + ); + + // Do longwave + rrtmgp_lw( + ncol, nlay, + k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + aerosol_lw, clouds_lw_gpt, + fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, + extra_clnclrsky_diag, extra_clnsky_diag + ); + +} + +/* + * Perform any clean-up tasks + */ +static void rrtmgp_finalize() +{ + initialized_k = false; + k_dist_sw_k.finalize(); + k_dist_lw_k.finalize(); + cloud_optics_sw_k.finalize(); //~CloudOptics(); + cloud_optics_lw_k.finalize(); //~CloudOptics(); + pool_t::finalize(); +} + +/* + * Shortwave driver (called by rrtmgp_main) + */ +template +static void rrtmgp_sw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, + OpticalProps2strK &aerosol, OpticalProps2strK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const RealT tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Get problem sizes + int nbnd = k_dist.get_nband(); + int ngpt = k_dist.get_ngpt(); + int ngas = gas_concs.get_num_gases(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &flux_dn_dir = fluxes.flux_dn_dir; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; + + // Reset fluxes to zero + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up (icol,ilev) = 0; + flux_dn (icol,ilev) = 0; + flux_dn_dir(icol,ilev) = 0; + clnclrsky_flux_up (icol,ilev) = 0; + clnclrsky_flux_dn (icol,ilev) = 0; + clnclrsky_flux_dn_dir(icol,ilev) = 0; + clrsky_flux_up (icol,ilev) = 0; + clrsky_flux_dn (icol,ilev) = 0; + clrsky_flux_dn_dir(icol,ilev) = 0; + clnsky_flux_up (icol,ilev) = 0; + clnsky_flux_dn (icol,ilev) = 0; + clnsky_flux_dn_dir(icol,ilev) = 0; + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up (icol,ilev,ibnd) = 0; + bnd_flux_dn (icol,ilev,ibnd) = 0; + bnd_flux_dn_dir(icol,ilev,ibnd) = 0; + }); + + // Get daytime indices + auto dayIndices = view_t("dayIndices", ncol); + Kokkos::deep_copy(dayIndices, -1); + + int nday = 0; + // Serialized for now. + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { + for (int icol = 0; icol < ncol; ++icol) { + if (mu0(icol) > 0) { + dayIndices(nday_inner++) = icol; + } + } + }, Kokkos::Sum(nday)); + + if (nday == 0) { + // No daytime columns in this chunk, skip the rest of this routine + return; + } + + // Subset mu0 + auto mu0_day = view_t("mu0_day", nday); + Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { + mu0_day(iday) = mu0(dayIndices(iday)); + }); + + // subset state variables + auto p_lay_day = view_t("p_lay_day", nday, nlay); + auto t_lay_day = view_t("t_lay_day", nday, nlay); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); + t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); + }); + auto p_lev_day = view_t("p_lev_day", nday, nlay+1); + auto t_lev_day = view_t("t_lev_day", nday, nlay+1); + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); + t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); + }); + + // Subset gases + auto gas_names = gas_concs.get_gas_names(); + GasConcsK gas_concs_day; + gas_concs_day.init(gas_names, nday, nlay); + for (int igas = 0; igas < ngas; igas++) { + auto vmr_day = view_t("vmr_day", nday, nlay); + auto vmr = view_t("vmr" , ncol, nlay); + gas_concs.get_vmr(gas_names[igas], vmr); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); + }); + gas_concs_day.set_vmr(gas_names[igas], vmr_day); + } + + // Subset aerosol optics + OpticalProps2strK aerosol_day; + aerosol_day.init(k_dist.get_band_lims_wavenumber()); + aerosol_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { + aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); + aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); + aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); + }); + + // Subset cloud optics + // TODO: nbnd -> ngpt once we pass sub-sampled cloud state + OpticalProps2strK clouds_day; + clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); + clouds_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { + clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); + clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); + clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); + }); + + // RRTMGP assumes surface albedos have a screwy dimension ordering + // for some strange reason, so we need to transpose these; also do + // daytime subsetting in the same kernel + view_t sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); + view_t sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); + Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { + sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); + sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); + }); + + // Temporaries we need for daytime-only fluxes + auto flux_up_day = view_t("flux_up_day", nday, nlay+1); + auto flux_dn_day = view_t("flux_dn_day", nday, nlay+1); + auto flux_dn_dir_day = view_t("flux_dn_dir_day", nday, nlay+1); + auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); + FluxesBybandK fluxes_day; + fluxes_day.flux_up = flux_up_day; + fluxes_day.flux_dn = flux_dn_day; + fluxes_day.flux_dn_dir = flux_dn_dir_day; + fluxes_day.bnd_flux_up = bnd_flux_up_day; + fluxes_day.bnd_flux_dn = bnd_flux_dn_day; + fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; + + // Allocate space for optical properties + OpticalProps2strK optics; + optics.alloc_2str(nday, nlay, k_dist); + + OpticalProps2strK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_2str(nday, nlay, k_dist); + } + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = view_t("t_lay_limited", nday, nlay); + limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); + + // Do gas optics + view_t toa_flux("toa_flux", nday, ngpt); + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); + if (extra_clnsky_diag) { + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); + check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); + check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); +#endif + + // Apply tsi_scaling + Kokkos::parallel_for(MDRP::template get<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { + toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); + }); + + if (extra_clnclrsky_diag) { + // Compute clear-clean-sky (just gas) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + + // Combine gas and aerosol optics + aerosol_day.delta_scale(); + aerosol_day.increment(optics); + + // Compute clearsky (gas + aerosol) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + + // Now merge in cloud optics and do allsky calculations + + // Combine gas and cloud optics + clouds_day.delta_scale(); + clouds_day.increment(optics); + // Compute fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + flux_up (icol,ilev) = flux_up_day (iday,ilev); + flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { + const int icol = dayIndices(iday); + bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); + bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); + bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); + }); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds_day.increment(optics_no_aerosols); + // Compute cleansky (gas + clouds) fluxes on daytime columns + rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } +} + +/* + * Longwave driver (called by rrtmgp_main) + */ +template +static void rrtmgp_lw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Problem size + int nbnd = k_dist.get_nband(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + + // Reset fluxes to zero + Kokkos::parallel_for( + MDRP::template get<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up(icol, ilev) = 0; + flux_dn(icol, ilev) = 0; + clnclrsky_flux_up(icol, ilev) = 0; + clnclrsky_flux_dn(icol, ilev) = 0; + clrsky_flux_up(icol, ilev) = 0; + clrsky_flux_dn(icol, ilev) = 0; + clnsky_flux_up(icol, ilev) = 0; + clnsky_flux_dn(icol, ilev) = 0; + }); + Kokkos::parallel_for( + MDRP::template get<3>({nbnd, nlay + 1, ncol}), + KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up(icol, ilev, ibnd) = 0; + bnd_flux_dn(icol, ilev, ibnd) = 0; + }); + + // Allocate space for optical properties + OpticalProps1sclK optics; + optics.alloc_1scl(ncol, nlay, k_dist); + OpticalProps1sclK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); + } + + // Boundary conditions + SourceFuncLWK lw_sources; + lw_sources.alloc(ncol, nlay, k_dist); + view_t t_sfc ("t_sfc" ,ncol); + view_t emis_sfc("emis_sfc",nbnd,ncol); + + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + // Surface temperature + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); + }); + Kokkos::deep_copy(emis_sfc , 0.98); + + // Get Gaussian quadrature weights + // TODO: move this crap out of userland! + // Weights and angle secants for first order (k=1) Gaussian quadrature. + // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 + // after Abramowitz & Stegun 1972, page 921 + int constexpr max_gauss_pts = 4; + hview_t gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; + gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; + gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; + gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; + + hview_t gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); + gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; + gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; + gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; + gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; + + view_t gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + view_t gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); + Kokkos::deep_copy(gauss_wts, gauss_wts_host); + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = view_t("t_lay_limited", ncol, nlay); + auto t_lev_limited = view_t("t_lev_limited", ncol, nlay+1); + limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); + limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); + + // Do gas optics + oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); + if (extra_clnsky_diag) { + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); +#endif + + if (extra_clnclrsky_diag) { + // Compute clean-clear-sky fluxes before we add in aerosols and clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); + } + + // Combine gas and aerosol optics + aerosol.increment(optics); + + // Compute clear-sky fluxes before we add in clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); + + // Combine gas and cloud optics + clouds.increment(optics); + + // Compute allsky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds.increment(optics_no_aerosols); + // Compute clean-sky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); + } +} + +/* + * Return a subcolumn mask consistent with a specified overlap assumption + */ +template +static view_t get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, + CldfT &cldf, const int overlap_option, SeedsT &seeds) +{ + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = view_t("subcolumn_mask", ncol, nlay, ngpt); + + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = view_t("cldx", ncol, nlay, ngpt); + + // Apply overlap assumption to set cldx + if (overlap_option == 0) { // Dummy mask, always cloudy + Kokkos::deep_copy(cldx, 1); + } else { // Default case, maximum-random overlap + // Maximum-random overlap: + // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, + // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same + // algorithm used in RRTMG implementation of maximum-random overlap (see + // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) + // + // First, fill cldx with random numbers. Need to use a unique seed for each column! + // auto seeds_host = Kokkos::create_mirror_view(seeds); + // Kokkos::deep_copy(seeds_host, seeds); + // for (int icol = 0; icol < ncol; ++icol) { + // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); + // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // auto generator = random_pool.get_state(); + // cldx(icol,ilay,igpt) = generator.drand(0., 1.); + // random_pool.free_state(generator); + // }); + // } + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + conv::Random rand(seeds(icol)); + for (int igpt = 0; igpt < ngpt; igpt++) { + for (int ilay = 0; ilay < nlay; ilay++) { + cldx(icol,ilay,igpt) = rand.genFP(); + } + } + }); + + // Step down columns and apply algorithm from eq (14) + Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + for (int ilay = 1; ilay < nlay; ilay++) { + // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn + if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { + // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent + // layers are maximimally overlapped + cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); + } else { + // Cloud-less above, use new random number so that clouds are distributed + // randomly in this layer. Need to scale new random number to range + // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution + // of random numbers in this layer with the above branch of the conditional, + // which would otherwise inflate cloud fraction in this layer. + cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); + } + } + }); + } + + // Use cldx array to create subcolumn mask + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { + subcolumn_mask(icol,ilay,igpt) = 1; + } else { + subcolumn_mask(icol,ilay,igpt) = 0; + } + }); + return subcolumn_mask; +} + +/* + * Compute cloud area from 3d subcol cloud property + */ +template +static void compute_cloud_area( + int ncol, int nlay, int ngpt, RealT pmin, RealT pmax, + const PmidT& pmid, const CldTauGptT& cld_tau_gpt, CldAreaT& cld_area) +{ + // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy + // then 2d subcol mask is 1, otherwise it is 0 + auto subcol_mask = view_t("subcol_mask", ncol, ngpt); + Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but + // using play/pmid does not + if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { + subcol_mask(icol,igpt) = 1; + } + }); + // Compute average over subcols to get cloud area + auto ngpt_inv = 1.0 / ngpt; + Kokkos::deep_copy(cld_area, 0); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // This loop needs to be serial because of the atomic reduction + for (int igpt = 0; igpt < ngpt; ++igpt) { + cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; + } + }); +} + +/* + * Return select cloud-top diagnostics following AeroCom recommendation + */ +template +static void compute_aerocom_cloudtop( + int ncol, int nlay, const TmidT &tmid, const PmidT &pmid, + const PdelT &p_del, const ZdelT &z_del, const QcT &qc, + const QiT &qi, const RelT &rel, const ReiT &rei, + const CldfracTotT &cldfrac_tot, const NcT &nc, + TmidAtCldtopT &T_mid_at_cldtop, PmidAtCldtopT &p_mid_at_cldtop, + CldfracIceAtCldtopT &cldfrac_ice_at_cldtop, CldfracLiqAtCldtopT &cldfrac_liq_at_cldtop, + CldfracTotAtCldtopT &cldfrac_tot_at_cldtop, CdncAtCldtopT &cdnc_at_cldtop, + EffRadiusQcAtCldtopT &eff_radius_qc_at_cldtop, EffRadiusQiAtCldTopT &eff_radius_qi_at_cldtop) +{ + /* The goal of this routine is to calculate properties at cloud top + * based on the AeroCom recommendation. See reference for routine + * get_subcolumn_mask above, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. + */ + // Set outputs to zero + Kokkos::deep_copy(T_mid_at_cldtop, 0.0); + Kokkos::deep_copy(p_mid_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); + Kokkos::deep_copy(cdnc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); + + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto aerocom_clr = view_t("aerocom_clr", ncol); + Kokkos::deep_copy(aerocom_clr, 1.0); + + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + + // TODO: move tunable constant to namelist + constexpr RealT q_threshold = 0.0; // BAD_CONSTANT! + + // TODO: move tunable constant to namelist + constexpr RealT cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! + + // Loop over all columns in parallel + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds + for(int ilay = 1; ilay < nlay; ++ilay) { + // Only do the calculation if certain conditions are met + if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && + (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + aerocom_clr(icol) * + (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), + cldfrac_tot(icol, ilay))) / + (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = + qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; + // p_mid_at_cldtop + p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + cldfrac_ice_at_cldtop(icol) += + (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / + z_del(icol, ilay) / physconst::gravit / + cldfrac_tot(icol, ilay); + cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtop + eff_radius_qc_at_cldtop(icol) += + rel(icol, ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtop + eff_radius_qi_at_cldtop(icol) += + rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + aerocom_clr(icol) = aerocom_tmp; + } + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); + }); +} + +/* + * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area + * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion + * with the standard practice of using "water path" to refer to the total column-integrated + * quantities). + */ template -void mixing_ratio_to_cloud_mass( +static void mixing_ratio_to_cloud_mass( View1 const& mixing_ratio, View2 const& cloud_fraction, View3 const& dp, @@ -315,7 +1174,7 @@ void mixing_ratio_to_cloud_mass( int ncol = mixing_ratio.extent(0); int nlay = mixing_ratio.extent(1); using physconst = scream::physics::Constants; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { // Compute in-cloud mixing ratio (mixing ratio of the cloudy part of the layer) // NOTE: these thresholds (from E3SM) seem arbitrary, but included here for consistency // This limits in-cloud mixing ratio to 0.005 kg/kg. According to note in cloud_diagnostics @@ -329,7 +1188,6 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif /* * Routine to limit a quantity to set bounds. Used to make sure @@ -337,33 +1195,199 @@ void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -#ifdef RRTMGP_ENABLE_YAKL template -void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { - yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { +static void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { + Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -template -void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + +static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) +{ + // Get wavelength bounds for all wavelength bands + auto wavelength_bounds = kdist.get_band_lims_wavelength(); + + // Find the band index for the specified wavelength + // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength + // in units of meters, we need a conversion factor of 10^2 + const int nbnds = kdist.get_nband(); + int band_index = -1; + Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { + if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { + if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } else { + if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } + }, Kokkos::Max(band_index)); + return band_index; +} + +static inline int get_wavelength_index_sw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_sw_k, wavelength); +} + +static inline int get_wavelength_index_lw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_lw_k, wavelength); +} + +template +static OpticalProps2strK get_cloud_optics_sw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + + // Initialize optics + OpticalProps2strK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_2str(ncol, nlay); + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = view_t("rel_limited", ncol, nlay); + auto rei_limited = view_t("rei_limited", ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} + +template +static OpticalProps1sclK get_cloud_optics_lw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + + // Initialize optics + OpticalProps1sclK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = view_t("rel_limited", ncol, nlay); + auto rei_limited = view_t("rei_limited", ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} + +template +OpticalProps2strK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { + // Initialized subsampled optics + OpticalProps2strK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_2str(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, + // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), + // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should + // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very + // high resolution. + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM + auto seeds = view_t("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); + subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + subsampled_optics.ssa(icol,ilay,igpt) = 0; + subsampled_optics.g (icol,ilay,igpt) = 0; + } }); + return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_YAKL -int get_wavelength_index(OpticalProps &kdist, double wavelength); -int get_wavelength_index_sw(double wavelength); -int get_wavelength_index_lw(double wavelength); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength); -int get_wavelength_index_sw_k(double wavelength); -int get_wavelength_index_lw_k(double wavelength); -#endif +template +OpticalProps1sclK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, + CldT &cld, PlayT &p_lay) { + // Initialized subsampled optics + OpticalProps1sclK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_1scl(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM; use different + // seed values for longwave and shortwave + auto seeds = view_t("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + } + }); + return subsampled_optics; +} + +}; // struct rrtmgp_interface +#endif // RRTMGP_ENABLE_KOKKOS } // namespace rrtmgp } // namespace scream From 75892307589e0f6734aa0501d2a439d95879731c Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 5 Jun 2024 16:32:50 -0700 Subject: [PATCH 019/477] first round of code clean up changes --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 61 ++++--------------- 1 file changed, 13 insertions(+), 48 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 6c7f41cf5972..da628df92e0f 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -338,28 +338,22 @@ apply_iop_forcing(const Real dt) // TODO: Create a memory buffer for this class // and add the below WSM and views WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); - WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); view_Nd rstar ("rstar", nelem, NGP, NGP, NLEV), exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing rstar, exner, and temperature from Hommexx + // Lambda for computing temperature from Hommexx auto compute_homme_states = [&] () { - Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("compute_temperature_for_iop", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { KV kv(team); const int ie = team.league_rank(); // Get temp views from workspace - auto ws = homme_wsm.get_workspace(team); - auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); - uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); - - // Get temp views from workspace - auto ws2 = eamxx_wsm.get_workspace(team); + auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid, pint, pdel; - ws2.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); + ws.take_many_contiguous_unsafe<1>({"pmid"}, + {&pmid}); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; @@ -368,49 +362,26 @@ apply_iop_forcing(const Real dt) auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto pnh_i = ekat::subview(pnh, idx); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); // Compute reference pressures and layer thickness. // TODO: Allow geometry data to allocate packsize auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; }); team.team_barrier(); // Reinterperate into views of Homme::Scalar for calling Hommexx function. Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); - Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - // Compute exner from EOS - if (theta_hydrostatic_mode) { - auto hydro_p_int = ws.take("hydro_p_int"); - Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); - elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); - eos.compute_exner(kv, pnh_scalar, exner_scalar); - ws.release(hydro_p_int); - } else { - eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); - } - - // Compute temperature + // Compute temperature from virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - auto& T_val = vtheta_dp_i(k); + auto T_val = vtheta_dp_i(k); T_val /= dp3d_i(k); T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); @@ -419,12 +390,11 @@ apply_iop_forcing(const Real dt) }); // Release WS views - ws.release_macro_block(pnh_slot, NGP*NGP); - ws2.release_many_contiguous<3>({&pmid, &pint, &pdel}); + ws.release_many_contiguous<1>({&pmid}); }); }; - // Preprocess some homme states to get temperature and exner + // Preprocess some homme states to get temperature compute_homme_states(); Kokkos::fence(); @@ -500,8 +470,6 @@ apply_iop_forcing(const Real dt) auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); // Compute reference pressures and layer thickness. @@ -517,10 +485,6 @@ apply_iop_forcing(const Real dt) team.team_barrier(); - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - // Compute Qdp from updated Q Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { const int ilev = k/qsize; @@ -532,7 +496,7 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - // Recompute rstar with updated qv, and convert updated temperature back to potential temperature + // Convert updated temperature back to psuedo density virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); @@ -661,6 +625,7 @@ apply_iop_forcing(const Real dt) temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); // Convert updated temperature back to potential temperature + // NEED TO REPLACE THIS DEFINITION! vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); } if (iop_nudge_uv) { From d3284e41b6bd0381f31f3ff1d9b94534712a4cfc Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 5 Jun 2024 17:10:47 -0700 Subject: [PATCH 020/477] update definition of vtheta_dp in IOP nudging --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index da628df92e0f..eff94346895e 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -352,8 +352,7 @@ apply_iop_forcing(const Real dt) // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<1>({"pmid"}, - {&pmid}); + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; @@ -594,10 +593,16 @@ apply_iop_forcing(const Real dt) KV kv(team); const int ie = team.league_rank(); + // Get temp views from workspace + auto ws = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; const int jgp = idx%NGP; + auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); auto rstar_i = ekat::subview(rstar, ie, igp, jgp); @@ -607,6 +612,14 @@ apply_iop_forcing(const Real dt) auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user @@ -624,9 +637,9 @@ apply_iop_forcing(const Real dt) qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to potential temperature - // NEED TO REPLACE THIS DEFINITION! - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); @@ -634,6 +647,8 @@ apply_iop_forcing(const Real dt) } }); }); + // Release WS views + ws.release_many_contiguous<1>({&pmid}); }); } } From 6b80ad2b9aded0b5b71b94df626a8066de3329e7 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 6 Jun 2024 16:55:23 -0600 Subject: [PATCH 021/477] Builds and runs --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 109 +++++----- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 130 ++++++------ .../src/physics/rrtmgp/rrtmgp_test_utils.cpp | 200 +----------------- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 128 ++++++++++- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 5 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 7 - .../rrtmgp/scream_rrtmgp_interface.hpp | 33 +-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 68 +++--- 9 files changed, 306 insertions(+), 376 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 59e6f72cc922..3d7fba5ac72d 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 59e6f72cc92268c69d1bd6ede570a959c1ebc28a +Subproject commit 3d7fba5ac72dab7c8448411e7452e2b409f22d93 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1ff3f27b2fce..a24f4d5ddc32 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -624,17 +624,17 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.init(gas_names_yakl_offset,m_col_chunk_size,m_nlay); - rrtmgp::rrtmgp_initialize( + interface_t::rrtmgp_initialize( m_gas_concs_k, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, m_atm_logger ); VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_sw, rrtmgp::k_dist_sw_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_lw, rrtmgp::k_dist_lw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, rrtmgp::cloud_optics_sw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, rrtmgp::cloud_optics_lw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_sw, interface_t::k_dist_sw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_lw, interface_t::k_dist_lw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, interface_t::cloud_optics_sw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, interface_t::cloud_optics_lw_k); #endif // Set property checks for fields in this process @@ -924,30 +924,30 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS // If YAKL is on, we don't want aliased memory in both the yakl and kokos // subviews. - auto subview_1dk = [&](const real1dk v) -> real1dk { - real1dk subv(v, std::make_pair(0, ncol)); + auto subview_1dk = [&](const view_1d_real v) -> view_1d_real { + view_1d_real subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - real1dk rv(v.label(), ncol); + view_1d_real rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_2dk = [&](const real2dk v) -> real2dk { - real2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_2dk = [&](const view_2d_real v) -> view_2d_real { + view_2d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); + view_2d_real rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_3dk = [&](const real3dk v) -> real3dk { - real3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + auto subview_3dk = [&](const view_3d_real v) -> view_3d_real { + view_3d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); + view_3d_real rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); return rv; #else @@ -1194,7 +1194,7 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::fence(); #ifdef RRTMGP_ENABLE_KOKKOS COMPARE_ALL_WRAP(std::vector({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1206,7 +1206,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real2d tmp2d = subview_2d(m_buffer.tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); + view_2d_real tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1308,14 +1308,14 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + interface_t::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + interface_t::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); + interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); + interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), - std::vector({lwp_k, iwp_k})); + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1350,13 +1350,13 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dir, sfc_alb_dif); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis_k, sfc_alb_dir_nir_k, sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), - std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1384,7 +1384,7 @@ void RRTMGPRadiation::run_impl (const double dt) { ); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, m_nlay, p_lay_k, t_lay_k, p_lev_k, t_lev_k, m_gas_concs_k, @@ -1412,7 +1412,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, lw_clnsky_flux_up, lw_clnsky_flux_dn}), - std::vector({ + std::vector({ sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, @@ -1422,7 +1422,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), - std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1472,7 +1472,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), - std::vector({sw_heating_k, lw_heating_k})); + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1509,14 +1509,14 @@ void RRTMGPRadiation::run_impl (const double dt) { sw_bnd_flux_dif_k(icol,ilev,ibnd) = sw_bnd_flux_dn_k(icol,ilev,ibnd) - sw_bnd_flux_dir_k(icol,ilev,ibnd); }); // Compute surface fluxes - rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot_k, nswbands, sw_bnd_flux_dir_k, sw_bnd_flux_dif_k, sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), - std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1537,22 +1537,22 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); - real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); - real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); - real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably // does not matter in practice, as clouds probably should not be produced above 50 hPa and we // should not be encountering surface pressure above 1200 hPa, but in the event that things go off // the rails we might want to look at these still. - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), - std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Compute cloud-top diagnostics following AeroCOM recommendation @@ -1580,20 +1580,20 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // Get visible 0.67 micron band for COSP - auto idx_067_k = rrtmgp::get_wavelength_index_sw_k(0.67e-6); + auto idx_067_k = interface_t::get_wavelength_index_sw_k(0.67e-6); // Get IR 10.5 micron band for COSP - auto idx_105_k = rrtmgp::get_wavelength_index_lw_k(10.5e-6); - - real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - - rrtmgp::compute_aerocom_cloudtop( + auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); + + view_1d_real T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + + interface_t::compute_aerocom_cloudtop( ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, @@ -1602,7 +1602,7 @@ void RRTMGPRadiation::run_impl (const double dt) { T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), - std::vector({ + std::vector({ T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); @@ -1699,7 +1699,7 @@ void RRTMGPRadiation::run_impl (const double dt) { }); #ifdef RRTMGP_ENABLE_YAKL // Sync back to gas_concs_k - real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + view_3d_real temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); Kokkos::deep_copy(temp, m_gas_concs_k.concs); #endif #endif @@ -1769,11 +1769,12 @@ void RRTMGPRadiation::run_impl (const double dt) { void RRTMGPRadiation::finalize_impl () { #ifdef RRTMGP_ENABLE_YAKL m_gas_concs.reset(); + rrtmgp::rrtmgp_finalize(); #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.reset(); + interface_t::rrtmgp_finalize(); #endif - rrtmgp::rrtmgp_finalize(); finalize_kls(); } diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 66c89476028b..28c5033c46d4 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -28,6 +28,10 @@ class RRTMGPRadiation : public AtmosphereProcess { template using uview_2d = Unmanaged>; + using layout_t = typename ekat::KokkosTypes::Layout; + + using interface_t = rrtmgp::rrtmgp_interface; + // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -92,7 +96,7 @@ class RRTMGPRadiation : public AtmosphereProcess { GasConcs m_gas_concs; #endif #ifdef RRTMGP_ENABLE_KOKKOS - GasConcsK m_gas_concs_k; + GasConcsK m_gas_concs_k; #endif // Prescribed greenhouse gas surface concentrations in moles / moles air @@ -137,15 +141,15 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + view_1d_real mu0_k; + view_1d_real sfc_alb_dir_vis_k; + view_1d_real sfc_alb_dir_nir_k; + view_1d_real sfc_alb_dif_vis_k; + view_1d_real sfc_alb_dif_nir_k; + view_1d_real sfc_flux_dir_vis_k; + view_1d_real sfc_flux_dir_nir_k; + view_1d_real sfc_flux_dif_vis_k; + view_1d_real sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) @@ -168,21 +172,21 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + view_2d_real p_lay_k; + view_2d_real t_lay_k; + view_2d_real z_del_k; + view_2d_real p_del_k; + view_2d_real qc_k; + view_2d_real nc_k; + view_2d_real qi_k; + view_2d_real cldfrac_tot_k; + view_2d_real eff_radius_qc_k; + view_2d_real eff_radius_qi_k; + view_2d_real tmp2d_k; + view_2d_real lwp_k; + view_2d_real iwp_k; + view_2d_real sw_heating_k; + view_2d_real lw_heating_k; #endif // 2d size (ncol, nlay+1) @@ -212,28 +216,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + view_2d_real p_lev_k; + view_2d_real t_lev_k; + view_2d_real sw_flux_up_k; + view_2d_real sw_flux_dn_k; + view_2d_real sw_flux_dn_dir_k; + view_2d_real lw_flux_up_k; + view_2d_real lw_flux_dn_k; + view_2d_real sw_clnclrsky_flux_up_k; + view_2d_real sw_clnclrsky_flux_dn_k; + view_2d_real sw_clnclrsky_flux_dn_dir_k; + view_2d_real sw_clrsky_flux_up_k; + view_2d_real sw_clrsky_flux_dn_k; + view_2d_real sw_clrsky_flux_dn_dir_k; + view_2d_real sw_clnsky_flux_up_k; + view_2d_real sw_clnsky_flux_dn_k; + view_2d_real sw_clnsky_flux_dn_dir_k; + view_2d_real lw_clnclrsky_flux_up_k; + view_2d_real lw_clnclrsky_flux_dn_k; + view_2d_real lw_clrsky_flux_up_k; + view_2d_real lw_clrsky_flux_dn_k; + view_2d_real lw_clnsky_flux_up_k; + view_2d_real lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -244,10 +248,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + view_3d_real sw_bnd_flux_up_k; + view_3d_real sw_bnd_flux_dn_k; + view_3d_real sw_bnd_flux_dir_k; + view_3d_real sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -256,8 +260,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + view_3d_real lw_bnd_flux_up_k; + view_3d_real lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -266,8 +270,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + view_2d_real sfc_alb_dir_k; + view_2d_real sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -278,10 +282,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + view_3d_real aero_tau_sw_k; + view_3d_real aero_ssa_sw_k; + view_3d_real aero_g_sw_k; + view_3d_real aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -290,8 +294,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + view_3d_real cld_tau_sw_bnd_k; + view_3d_real cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -300,8 +304,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + view_3d_real cld_tau_sw_gpt_k; + view_3d_real cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp index 54a32133da92..65cc99e09cc3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp @@ -1,25 +1,13 @@ -#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" #include "physics/rrtmgp/rrtmgp_test_utils.hpp" #ifdef RRTMGP_ENABLE_YAKL #include "YAKL_netcdf.h" #endif -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rte/mo_fluxes.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" - #include #include namespace rrtmgpTest { -#ifdef RRTMGP_ENABLE_YAKL -using yakl::fortran::parallel_for; -using yakl::fortran::SimpleBounds; -using yakl::intrinsics::mod; -using yakl::intrinsics::merge; -#endif - bool file_exists(const char *filename) { if (auto file = fopen(filename, "r")) { fclose(file); @@ -29,8 +17,12 @@ bool file_exists(const char *filename) { } } -// TODO: use YAKL intrinsics for this to avoid needing to make host copies #ifdef RRTMGP_ENABLE_YAKL +using yakl::fortran::parallel_for; +using yakl::fortran::SimpleBounds; +using yakl::intrinsics::mod; +using yakl::intrinsics::merge; + bool all_close(real2d &arr1, real2d &arr2, double tolerance) { int nx = arr1.dimension[0]; int ny = arr2.dimension[1]; @@ -46,28 +38,7 @@ bool all_close(real2d &arr1, real2d &arr2, double tolerance) { } return true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { - int nx = arr1.extent(0); - int ny = arr2.extent(1); - auto arr1_h = Kokkos::create_mirror_view(arr1); - auto arr2_h = Kokkos::create_mirror_view(arr2); - Kokkos::deep_copy(arr1_h, arr1); - Kokkos::deep_copy(arr2_h, arr2); - for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { - printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); - return false; - } - } - } - return true; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void dummy_atmos( std::string inputfile, int ncol, real2d &p_lay, real2d &t_lay, @@ -94,37 +65,7 @@ void dummy_atmos( // create a dummy atmosphere. dummy_clouds(scream::rrtmgp::cloud_optics_sw, p_lay, t_lay, lwp, iwp, rel, rei, cld); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_atmos( - std::string inputfile, - int ncol, real2dk &p_lay, real2dk &t_lay, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld) { - - // Setup boundary conditions, solar zenith angle, etc - // NOTE: this stuff would come from the model in a real run - // Ocean-ish values for surface albedos, just for example - Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); - - // Pick a solar zenith angle; this should come from the model - Kokkos::deep_copy(mu0, 0.86 ); - - // Get dummy cloud PHYSICAL properties. Note that this function call - // needs the CloudOptics object only because it uses the min and max - // valid values from the lookup tables for liquid and ice water path to - // create a dummy atmosphere. - dummy_clouds(scream::rrtmgp::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void dummy_clouds( CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cloud_mask) { @@ -151,38 +92,7 @@ void dummy_clouds( rei(icol,ilay) = merge(rei_val, 0._wp, iwp(icol,ilay) > 0._wp); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask) { - - // Problem sizes - int ncol = t_lay.extent(0); - int nlay = t_lay.extent(1); - - // Generate some fake liquid and ice water data. We pick values to be midway between - // the min and max of the valid lookup table values for effective radii - real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); - real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); - // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and - // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. - // Set sane values for liquid and ice water path. - // NOTE: these "sane" values are in g/m2! - Kokkos::parallel_for( conv::get_mdrp<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { - cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; - // Ice and liquid will overlap in a few layers - lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); - iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); - rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); - rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); - }); -} -#endif - -// Function to read fluxes from input file so we can compare our answers against the reference -#ifdef RRTMGP_ENABLE_YAKL void read_fluxes( std::string inputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -208,36 +118,7 @@ void read_fluxes( io.read(lw_flux_up, "lw_flux_up" ); io.read(lw_flux_dn, "lw_flux_dn" ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void read_fluxes( - std::string inputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - // Initialize netcdf reader - conv::SimpleNetCDF io; - io.open(inputfile, NC_NOWRITE); - - // Initialize arrays to hold fluxes - int nlev = io.getDimSize("lev"); - int ncol = io.getDimSize("col_flx"); - sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); - sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); - sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); - lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); - lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); - - // Read data - io.read(sw_flux_up, "sw_flux_up" ); - io.read(sw_flux_dn, "sw_flux_dn" ); - io.read(sw_flux_dir, "sw_flux_dir"); - io.read(lw_flux_up, "lw_flux_up" ); - io.read(lw_flux_dn, "lw_flux_dn" ); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void write_fluxes( std::string outputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -253,76 +134,5 @@ void write_fluxes( io.close(); } #endif -#ifdef RRTMGP_ENABLE_KOKKOS -void write_fluxes( - std::string outputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - conv::SimpleNetCDF io; - io.create(outputfile); - io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); - io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); - io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); - io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); - io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); - io.close(); -} -#endif - -// TODO: This function should instead take values, not file names, -// because for the test we do not want to write to file -#ifdef RRTMGP_ENABLE_YAKL -int compare_y(std::string file1, std::string file2) { - // Read data from baseline and test file - real2d sw_flux_up_1; - real2d sw_flux_dn_1; - real2d sw_flux_dir_1; - real2d lw_flux_up_1; - real2d lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2d sw_flux_up_2; - real2d sw_flux_dn_2; - real2d sw_flux_dir_2; - real2d lw_flux_up_2; - real2d lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int compare_k(std::string file1, std::string file2) { - // Read data from baseline and test file - real2dk sw_flux_up_1; - real2dk sw_flux_dn_1; - real2dk sw_flux_dir_1; - real2dk lw_flux_up_1; - real2dk lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2dk sw_flux_up_2; - real2dk sw_flux_dn_2; - real2dk sw_flux_dir_2; - real2dk lw_flux_up_2; - real2dk lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index b9498a9a34f9..79f6e570950d 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -1,5 +1,10 @@ #ifndef RRTMGP_TEST_UTILS_HPP #define RRTMGP_TEST_UTILS_HPP + +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rte/mo_fluxes.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" namespace rrtmgpTest { @@ -37,33 +42,134 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance); +template +struct rrtmgp_test_utils { -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +using real1dk = typename ekat::KokkosTypes::template view_1d; +using real2dk = typename ekat::KokkosTypes::template view_2d; +using real3dk = typename ekat::KokkosTypes::template view_3d; +using MDRP = typename conv::MDRP; +using interface_t = scream::rrtmgp::rrtmgp_interface; -void dummy_atmos( +static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) +{ + int nx = arr1.extent(0); + int ny = arr2.extent(1); + auto arr1_h = Kokkos::create_mirror_view(arr1); + auto arr2_h = Kokkos::create_mirror_view(arr2); + Kokkos::deep_copy(arr1_h, arr1); + Kokkos::deep_copy(arr2_h, arr2); + for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { + printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); + return false; + } + } + } + return true; +} + +static void dummy_clouds( + CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask +) +{ + // Problem sizes + int ncol = t_lay.extent(0); + int nlay = t_lay.extent(1); + + // Generate some fake liquid and ice water data. We pick values to be midway between + // the min and max of the valid lookup table values for effective radii + real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); + real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); + + // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and + // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. + // Set sane values for liquid and ice water path. + // NOTE: these "sane" values are in g/m2! + Kokkos::parallel_for( MDRP::template get<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { + cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; + // Ice and liquid will overlap in a few layers + lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); + iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); + rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); + rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); + }); +} + +static void dummy_atmos( std::string inputfile, int ncol, real2dk &p_lay, real2dk &t_lay, real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, real1dk &mu0, real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +) +{ + // Setup boundary conditions, solar zenith angle, etc + // NOTE: this stuff would come from the model in a real run -void read_fluxes( + // Ocean-ish values for surface albedos, just for example + Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); + + // Pick a solar zenith angle; this should come from the model + Kokkos::deep_copy(mu0, 0.86 ); + + // Get dummy cloud PHYSICAL properties. Note that this function call + // needs the CloudOptics object only because it uses the min and max + // valid values from the lookup tables for liquid and ice water path to + // create a dummy atmosphere. + dummy_clouds(interface_t::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); +} + +static void read_fluxes( std::string inputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + // Initialize netcdf reader + conv::SimpleNetCDF io; + io.open(inputfile, NC_NOWRITE); -void write_fluxes( + // Initialize arrays to hold fluxes + int nlev = io.getDimSize("lev"); + int ncol = io.getDimSize("col_flx"); + sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); + sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); + sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); + lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); + lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); + + // Read data + io.read(sw_flux_up, "sw_flux_up" ); + io.read(sw_flux_dn, "sw_flux_dn" ); + io.read(sw_flux_dir, "sw_flux_dir"); + io.read(lw_flux_up, "lw_flux_up" ); + io.read(lw_flux_dn, "lw_flux_dn" ); +} + +static void write_fluxes( std::string outputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + conv::SimpleNetCDF io; + io.create(outputfile); + io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); + io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); + io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); + io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); + io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); + io.close(); +} + +}; #endif } diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 119093d5f9d3..6e924103229f 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -109,8 +109,9 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, RealT xmin, RealT xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 1696e7933685..10281df8078c 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,11 +1,4 @@ #include "scream_rrtmgp_interface.hpp" -#include "cpp/examples/mo_load_coefficients.h" -#include "examples/all-sky/mo_load_cloud_coefficients.h" -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" -#include "cpp/rte/mo_rte_sw.h" -#include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" namespace scream { diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index c40be4904302..2038cb62ff79 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -4,7 +4,14 @@ #include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" +#include "cpp/examples/mo_load_coefficients.h" #include "cpp/rrtmgp_const.h" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "cpp/rte/mo_rte_sw.h" +#include "cpp/rte/mo_rte_lw.h" +#include "examples/all-sky/mo_load_cloud_coefficients.h" #include "rrtmgp_utils.hpp" @@ -152,7 +159,7 @@ int get_wavelength_index_lw(double wavelength); // New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_interface { using MDRP = typename conv::MDRP; @@ -469,8 +476,8 @@ static void rrtmgp_main( #endif // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); @@ -640,7 +647,7 @@ static void rrtmgp_sw( // Subset gases auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; + GasConcsK gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { auto vmr_day = view_t("vmr_day", nday, nlay); @@ -653,7 +660,7 @@ static void rrtmgp_sw( } // Subset aerosol optics - OpticalProps2strK aerosol_day; + OpticalProps2strK aerosol_day; aerosol_day.init(k_dist.get_band_lims_wavenumber()); aerosol_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { @@ -664,7 +671,7 @@ static void rrtmgp_sw( // Subset cloud optics // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; + OpticalProps2strK clouds_day; clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); clouds_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { @@ -690,7 +697,7 @@ static void rrtmgp_sw( auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; + FluxesBybandK fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; fluxes_day.flux_dn_dir = flux_dn_dir_day; @@ -699,10 +706,10 @@ static void rrtmgp_sw( fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; // Allocate space for optical properties - OpticalProps2strK optics; + OpticalProps2strK optics; optics.alloc_2str(nday, nlay, k_dist); - OpticalProps2strK optics_no_aerosols; + OpticalProps2strK optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_2str(nday, nlay, k_dist); @@ -1195,8 +1202,8 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template -static void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { +template +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); @@ -1291,7 +1298,7 @@ static OpticalProps1sclK get_cloud_optics_lw( } template -OpticalProps2strK get_subsampled_clouds( +static OpticalProps2strK get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { // Initialized subsampled optics @@ -1342,7 +1349,7 @@ OpticalProps2strK get_subsampled_clouds( } template -OpticalProps1sclK get_subsampled_clouds( +static OpticalProps1sclK get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index ddef61808fe0..36e2a59e02ff 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -849,6 +849,14 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { #endif #ifdef RRTMGP_ENABLE_KOKKOS +using interface_t = scream::rrtmgp::rrtmgp_interface<>; +using real1dk = interface_t::view_t; +using real2dk = interface_t::view_t; +using real3dk = interface_t::view_t; +using int1dk = interface_t::view_t; +using int2dk = interface_t::view_t; +using int3dk = interface_t::view_t; + TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos scream::init_kls(); @@ -921,7 +929,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 1.0; }); auto cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with no cloud @@ -931,7 +939,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.0; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with empty clouds (cloud fraction but with no associated mixing ratio) @@ -943,7 +951,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.1; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with cell half filled with cloud @@ -953,7 +961,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.5; }); cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up @@ -977,14 +985,14 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { }); // Limit to bounds that contain the data; should be no change in values - scream::rrtmgp::limit_to_bounds(arr, 0.0, 5.0, arr_limited); + interface_t::limit_to_bounds_k(arr, 0.0, 5.0, arr_limited); REQUIRE(chc(arr)(0,0) == chc(arr_limited)(0,0)); REQUIRE(chc(arr)(0,1) == chc(arr_limited)(0,1)); REQUIRE(chc(arr)(1,0) == chc(arr_limited)(1,0)); REQUIRE(chc(arr)(1,1) == chc(arr_limited)(1,1)); // Limit to bounds that do not completely contain the data; should be a change in values! - scream::rrtmgp::limit_to_bounds(arr, 1.5, 3.5, arr_limited); + interface_t::limit_to_bounds_k(arr, 1.5, 3.5, arr_limited); REQUIRE(chc(arr_limited)(0,0) == 1.5); REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); @@ -1078,7 +1086,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Create simple test cases; We expect, given the input data, that band 10 // will straddle the NIR and VIS, bands 1-9 will be purely NIR, and bands 11-14 @@ -1106,7 +1114,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1138,7 +1146,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1169,7 +1177,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1200,7 +1208,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1216,7 +1224,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Finalize YAKL logger->info("Free memory...\n"); - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); gas_concs.reset(); scream::finalize_kls(); } @@ -1282,7 +1290,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { @@ -1316,7 +1324,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); auto cldmask_h = chc(cldmask); for (int igpt = 0; igpt < ngpt; igpt++) { if (cldmask_h(0,0,igpt) == 1) { @@ -1359,7 +1367,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); // Case: @@ -1376,7 +1384,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 1; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); // Case: @@ -1393,11 +1401,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1.0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0 / 3.0); // Case: @@ -1414,11 +1422,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); scream::finalize_kls(); } @@ -1463,7 +1471,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { Kokkos::deep_copy(rel, 10.0); Kokkos::deep_copy(rei, 10.0); // Call the function - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1481,7 +1489,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { // Case 2: if all clouds, everything goes to 1 * its value Kokkos::deep_copy(cldfrac_tot, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1504,7 +1512,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 3) = 0.3; cldfrac_tot(0, 4) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1520,7 +1528,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 5) = 0.4; cldfrac_tot(0, 6) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1534,7 +1542,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 4) = 0.0; cldfrac_tot(0, 5) = 0.1; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1550,7 +1558,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 1.0); Kokkos::deep_copy(qi, 0.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1568,7 +1576,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 0.0); Kokkos::deep_copy(qi, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1604,7 +1612,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { qc(0, 6) = 50; qc(0, 7) = 10; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, From df2d2da92a5fdb59baa02088fe5a04dbd3f42cf1 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 6 Jun 2024 16:10:58 -0700 Subject: [PATCH 022/477] remove exner and rstar in homme iop as they are no longer needed and fix formatting issues --- .../eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index eff94346895e..81765e2c6a7e 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -339,8 +339,6 @@ apply_iop_forcing(const Real dt) // and add the below WSM and views WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); view_Nd - rstar ("rstar", nelem, NGP, NGP, NLEV), - exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); // Lambda for computing temperature from Hommexx @@ -381,9 +379,9 @@ apply_iop_forcing(const Real dt) // Compute temperature from virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); }); }); @@ -498,7 +496,7 @@ apply_iop_forcing(const Real dt) // Convert updated temperature back to psuedo density virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); // Release WS views @@ -511,7 +509,7 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute rstar, exner and temperature from Hommexx + // Compute temperature from Hommexx compute_homme_states(); Kokkos::fence(); } @@ -605,8 +603,6 @@ apply_iop_forcing(const Real dt) auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); From 0819dd38967299ec94e4ff11656998fc02a7a41b Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 10 Jun 2024 09:30:25 -0700 Subject: [PATCH 023/477] address a minor formatting issue --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 81765e2c6a7e..ee76f7229bbc 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -356,7 +356,7 @@ apply_iop_forcing(const Real dt) const int igp = idx/NGP; const int jgp = idx%NGP; - auto ps_i = ps_dyn(ie, igp, jgp); + auto ps_i = ps_dyn(ie, igp, jgp); auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); From 771fa4ee77919b0a102e0e92e6e45ffe1f254a00 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 09:10:32 -0700 Subject: [PATCH 024/477] clean up of unnecessary code --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 65b51b4c11be..22e0add1fe99 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -341,15 +341,15 @@ apply_iop_forcing(const Real dt) view_Nd temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing temperature from Hommexx - auto compute_homme_states = [&] () { + // Lambda for computing temperature + auto compute_temperature = [&] () { Kokkos::parallel_for("compute_temperature_for_iop", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { KV kv(team); const int ie = team.league_rank(); // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; + uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { @@ -370,12 +370,6 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); - Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - // Compute temperature from virtual potential temperature Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { auto T_val = vtheta_dp_i(k); @@ -392,7 +386,7 @@ apply_iop_forcing(const Real dt) }; // Preprocess some homme states to get temperature - compute_homme_states(); + compute_temperature(); Kokkos::fence(); // Apply IOP forcing @@ -509,8 +503,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute temperature from Hommexx - compute_homme_states(); + // Compute temperature + compute_temperature(); Kokkos::fence(); } @@ -593,7 +587,7 @@ apply_iop_forcing(const Real dt) // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; + uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { From 5145ccf2f4fd5f14d717cb100a1f3e8b21887424 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 10:28:02 -0700 Subject: [PATCH 025/477] change two instances of policy_homme to policy_eamxx --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 133 ++++++++---------- 1 file changed, 62 insertions(+), 71 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 22e0add1fe99..4bff0c0fa1ee 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -343,41 +343,36 @@ apply_iop_forcing(const Real dt) // Lambda for computing temperature auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + Kokkos::parallel_for("compute_temperature_for_iop", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); - }); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + // Compute temperature from virtual potential temperature + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + auto T_val = vtheta_dp_i(k); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); }); // Release WS views @@ -441,61 +436,57 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; - // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); + // Get temp views from workspace + auto ws = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); - team.team_barrier(); + team.team_barrier(); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - }); + // Convert updated temperature back to psuedo density virtual potential temperature + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + }); // Release WS views ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); - }); }); if (iop_nudge_tq or iop_nudge_uv) { From f9e7595585ce0825f0eb3270657b77a05134e138 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 11:10:13 -0700 Subject: [PATCH 026/477] last change from policy_homme to policy_eamxx in the nudging routine --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 98 +++++++++---------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 4bff0c0fa1ee..93cddb99877d 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -571,65 +571,63 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_homme, + policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; // Get temp views from workspace auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { - if (iop_nudge_tq) { - // Restrict nudging of T and qv to certain levels if requested by user - // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high - // is in units of [hPa], thus convert iop_nudge_tq_low/high - Mask nudge_level(false); - int max_size = hyam.size(); - for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { - const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; - nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); - } - - qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); - temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - } - if (iop_nudge_uv) { - u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); - v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { + if (iop_nudge_tq) { + // Restrict nudging of T and qv to certain levels if requested by user + // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high + // is in units of [hPa], thus convert iop_nudge_tq_low/high + Mask nudge_level(false); + int max_size = hyam.size(); + for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { + const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; + nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 + and + pressure_from_iop >= iop_nudge_tq_high*100); } - }); + + qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); + temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); + + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + } + if (iop_nudge_uv) { + u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); + v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); + } }); - // Release WS views - ws.release_many_contiguous<1>({&pmid}); + + // Release WS views + ws.release_many_contiguous<1>({&pmid}); }); } } From 8731c480f22bb64a18737d3e43cf364dd6d2bf87 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 11:38:11 -0700 Subject: [PATCH 027/477] remove unnecessary includes statements --- .../eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 93cddb99877d..b37985608f73 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,11 +10,8 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" -#include "ElementOps.hpp" -#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" -#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" @@ -225,13 +222,7 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - - using EOS = Homme::EquationOfState; - using ElementOps = Homme::ElementOps; - using KV = Homme::KernelVariables; - using PF = PhysicsFunctions; - using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -268,7 +259,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states and EOS/EO classes + // Homme element states auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -277,11 +268,6 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - EOS eos; - eos.init(params.theta_hydrostatic_mode, hvcoord); - - ElementOps elem_ops; - elem_ops.init(hvcoord); const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); // Load data from IOP files, if necessary From 09cfa6a9d896e0e3cf574daa75e71b9f7cc0f0b9 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 12 Jun 2024 11:43:14 -0700 Subject: [PATCH 028/477] remove lingering definition of policy_homme and update comment --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index b37985608f73..77330878f21a 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -310,15 +310,10 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for both homme and scream - // related loops. We need separate policies since hommexx functions used here - // assume they are called inside nested loops for elements and Gaussian points, - // whereas EAMxx function we use expects a single level of parallelism - // for elements and Guassian points. + // Team policy and workspace manager for eamxx // TODO: scream::ColumnOps functions could take an arbitary loop boundary // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that // all 3 kernel launches here could be combined. - const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class From f8211b93bf9de0f6544a523ba7cd7d7e2cc394d4 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 17 Jun 2024 09:44:20 -0700 Subject: [PATCH 029/477] rename policy and workspaces and change a few instances of improper ThreadVectorRange to TeamVectorRange --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 77330878f21a..37e6c9de8adc 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -311,26 +311,23 @@ apply_iop_forcing(const Real dt) } // Team policy and workspace manager for eamxx - // TODO: scream::ColumnOps functions could take an arbitary loop boundary - // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that - // all 3 kernel launches here could be combined. - const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); + WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); view_Nd temperature("temperature", nelem, NGP, NGP, NLEV); // Lambda for computing temperature auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); @@ -349,7 +346,7 @@ apply_iop_forcing(const Real dt) team.team_barrier(); // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { auto T_val = vtheta_dp_i(k); T_val /= dp3d_i(k); T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); @@ -366,13 +363,13 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -417,13 +414,13 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -450,7 +447,7 @@ apply_iop_forcing(const Real dt) team.team_barrier(); // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { const int ilev = k/qsize; const int q = k%qsize; @@ -461,7 +458,7 @@ apply_iop_forcing(const Real dt) team.team_barrier(); // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); @@ -552,7 +549,7 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_eamxx, + policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); @@ -560,7 +557,7 @@ apply_iop_forcing(const Real dt) const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid; ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); From eac5b2f2ea2d4694a5bd13c353e28ba9b4d3b9aa Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 14:15:42 -0600 Subject: [PATCH 030/477] Everything working --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 327 +++++++++--------- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 158 ++++----- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 10 +- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 89 ++++- .../rrtmgp/scream_rrtmgp_interface.hpp | 24 +- .../src/physics/rrtmgp/tests/rrtmgp_tests.cpp | 64 ++-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 15 +- .../rrtmgp/rrtmgp_standalone_unit.cpp | 37 +- 9 files changed, 405 insertions(+), 321 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 3d7fba5ac72d..4c542032b3ac 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 3d7fba5ac72dab7c8448411e7452e2b409f22d93 +Subproject commit 4c542032b3ac3f842a9619c9eadc005c507a0366 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index a24f4d5ddc32..1556280eab4b 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -19,6 +19,28 @@ namespace scream { +template +void check_views2(const View1& view1, const View2& view2) +{ + constexpr auto krank1 = View1::rank; + constexpr auto krank2 = View2::rank; + + RRT_REQUIRE(krank1 == krank2, "Rank mismatch for: " << view1.label()); + RRT_REQUIRE(krank1 == 2, "Rank mismatch for: " << view1.label()); + + for (auto r = 0; r < krank1; ++r) { + RRT_REQUIRE(view1.extent(r) == view2.extent(r), "Dim mismatch for: " << view1.label() << ", rank: " << r << ", " << view1.extent(r) << " != " << view2.extent(r)); + } + + for (auto i = 0; i < view1.extent(0); ++i) { + for (auto j = 0; j < view1.extent(1); ++j) { + const auto data1 = view1(i, j); + const auto data2 = view2(i, j); + RRT_REQUIRE(data1 == data2, "Data mismatch for: " << view1.label() << ", i: " << i << ", j: " << j << ", " << data1 << " != " << data2); + } + } +} + using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; @@ -596,7 +618,7 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // Names of active gases auto gas_names_yakl_offset = string1dv(m_ngas); - m_gas_mol_weights = view_1d_real("gas_mol_weights",m_ngas); + m_gas_mol_weights = real1dk("gas_mol_weights",m_ngas); // the lookup function for getting the gas mol weights doesn't work on device auto gas_mol_w_host = Kokkos::create_mirror_view(m_gas_mol_weights); for (int igas = 0; igas < m_ngas; igas++) { @@ -848,11 +870,9 @@ void RRTMGPRadiation::run_impl (const double dt) { "[RRTMGP::run_impl] Col chunk beg,end: " + std::to_string(beg) + ", " + std::to_string(beg+ncol) + "\n"); +#ifdef RRTMGP_ENABLE_YAKL // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. - // JGF: this doesn't appear to be copying the data, just returning a new array - // pointing to the same memory. -#ifdef RRTMGP_ENABLE_YAKL auto subview_1d = [&](const real1d v) -> real1d { return real1d(v.label(),v.myData,ncol); }; @@ -923,31 +943,64 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // If YAKL is on, we don't want aliased memory in both the yakl and kokos - // subviews. - auto subview_1dk = [&](const view_1d_real v) -> view_1d_real { - view_1d_real subv(v, std::make_pair(0, ncol)); + // subviews, so make new views and deep_copy. Also, be sure to trim the excess + // items from the field manager views due to simd packs. If we don't trim, then + // check_range_k will fail due to looking at unused values. Once rrtmgp can handle + // packs, this won't be necessary. + auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { + ureal1dk subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - view_1d_real rv(v.label(), ncol); + real1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_2dk = [&](const view_2d_real v) -> view_2d_real { - view_2d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { + cureal1dk subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - view_2d_real rv(v.label(), ncol, v.extent(1)); + creal1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_3dk = [&](const view_3d_real v) -> view_3d_real { - view_3d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + auto subview_2dk = [&](const ureal2dk& v) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - view_3d_real rv(v.label(), ncol, v.extent(1), v.extent(2)); + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_2dkc = [&](const cureal2dk& v, const int inner_dim) -> cureal2dk { + cureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); +#ifdef RRTMGP_ENABLE_YAKL + creal2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_2dk_new = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); +#ifdef RRTMGP_ENABLE_YAKL + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { + ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); +#ifdef RRTMGP_ENABLE_YAKL + real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); return rv; #else @@ -955,59 +1008,59 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; - auto p_lay_k = subview_2dk(m_buffer.p_lay_k); - auto t_lay_k = subview_2dk(m_buffer.t_lay_k); - auto p_lev_k = subview_2dk(m_buffer.p_lev_k); - auto z_del_k = subview_2dk(m_buffer.z_del_k); - auto p_del_k = subview_2dk(m_buffer.p_del_k); - auto t_lev_k = subview_2dk(m_buffer.t_lev_k); - auto mu0_k = subview_1dk(m_buffer.mu0_k); - auto sfc_alb_dir_k = subview_2dk(m_buffer.sfc_alb_dir_k); - auto sfc_alb_dif_k = subview_2dk(m_buffer.sfc_alb_dif_k); - auto sfc_alb_dir_vis_k = subview_1dk(m_buffer.sfc_alb_dir_vis_k); - auto sfc_alb_dir_nir_k = subview_1dk(m_buffer.sfc_alb_dir_nir_k); - auto sfc_alb_dif_vis_k = subview_1dk(m_buffer.sfc_alb_dif_vis_k); - auto sfc_alb_dif_nir_k = subview_1dk(m_buffer.sfc_alb_dif_nir_k); - auto qc_k = subview_2dk(m_buffer.qc_k); - auto nc_k = subview_2dk(m_buffer.nc_k); - auto qi_k = subview_2dk(m_buffer.qi_k); - auto cldfrac_tot_k = subview_2dk(m_buffer.cldfrac_tot_k); - auto rel_k = subview_2dk(m_buffer.eff_radius_qc_k); - auto rei_k = subview_2dk(m_buffer.eff_radius_qi_k); - auto sw_flux_up_k = subview_2dk(m_buffer.sw_flux_up_k); - auto sw_flux_dn_k = subview_2dk(m_buffer.sw_flux_dn_k); - auto sw_flux_dn_dir_k = subview_2dk(m_buffer.sw_flux_dn_dir_k); - auto lw_flux_up_k = subview_2dk(m_buffer.lw_flux_up_k); - auto lw_flux_dn_k = subview_2dk(m_buffer.lw_flux_dn_k); - auto sw_clnclrsky_flux_up_k = subview_2dk(m_buffer.sw_clnclrsky_flux_up_k); - auto sw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_k); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_dir_k); - auto sw_clrsky_flux_up_k = subview_2dk(m_buffer.sw_clrsky_flux_up_k); - auto sw_clrsky_flux_dn_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_k); - auto sw_clrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_dir_k); - auto sw_clnsky_flux_up_k = subview_2dk(m_buffer.sw_clnsky_flux_up_k); - auto sw_clnsky_flux_dn_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_k); - auto sw_clnsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_dir_k); - auto lw_clnclrsky_flux_up_k = subview_2dk(m_buffer.lw_clnclrsky_flux_up_k); - auto lw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.lw_clnclrsky_flux_dn_k); - auto lw_clrsky_flux_up_k = subview_2dk(m_buffer.lw_clrsky_flux_up_k); - auto lw_clrsky_flux_dn_k = subview_2dk(m_buffer.lw_clrsky_flux_dn_k); - auto lw_clnsky_flux_up_k = subview_2dk(m_buffer.lw_clnsky_flux_up_k); - auto lw_clnsky_flux_dn_k = subview_2dk(m_buffer.lw_clnsky_flux_dn_k); - auto sw_bnd_flux_up_k = subview_3dk(m_buffer.sw_bnd_flux_up_k); - auto sw_bnd_flux_dn_k = subview_3dk(m_buffer.sw_bnd_flux_dn_k); - auto sw_bnd_flux_dir_k = subview_3dk(m_buffer.sw_bnd_flux_dir_k); - auto sw_bnd_flux_dif_k = subview_3dk(m_buffer.sw_bnd_flux_dif_k); - auto lw_bnd_flux_up_k = subview_3dk(m_buffer.lw_bnd_flux_up_k); - auto lw_bnd_flux_dn_k = subview_3dk(m_buffer.lw_bnd_flux_dn_k); - auto sfc_flux_dir_vis_k = subview_1dk(m_buffer.sfc_flux_dir_vis_k); - auto sfc_flux_dir_nir_k = subview_1dk(m_buffer.sfc_flux_dir_nir_k); - auto sfc_flux_dif_vis_k = subview_1dk(m_buffer.sfc_flux_dif_vis_k); - auto sfc_flux_dif_nir_k = subview_1dk(m_buffer.sfc_flux_dif_nir_k); - auto aero_tau_sw_k = subview_3dk(m_buffer.aero_tau_sw_k); - auto aero_ssa_sw_k = subview_3dk(m_buffer.aero_ssa_sw_k); - auto aero_g_sw_k = subview_3dk(m_buffer.aero_g_sw_k); - auto aero_tau_lw_k = subview_3dk(m_buffer.aero_tau_lw_k); + // Note, ncol will not necessary be m_col_chunk_size because the number of cols + // will not always be evenly divided by m_col_chunk_size. In most cases, the + // extra space will not cause any problems, but it does sometimes. + auto p_lay_k = subview_2dkc(d_pmid, m_nlay); + auto t_lay_k = subview_2dkc(d_tmid, m_nlay); + auto p_lev_k = subview_2dkc(d_pint, m_nlay+1); + auto p_del_k = subview_2dkc(d_pdel, m_nlay); + auto sfc_alb_dir_k = m_buffer.sfc_alb_dir_k; + auto sfc_alb_dif_k = m_buffer.sfc_alb_dif_k; + auto sfc_alb_dir_vis_k = subview_1dkc(d_sfc_alb_dir_vis); + auto sfc_alb_dir_nir_k = subview_1dkc(d_sfc_alb_dir_nir); + auto sfc_alb_dif_vis_k = subview_1dkc(d_sfc_alb_dif_vis); + auto sfc_alb_dif_nir_k = subview_1dkc(d_sfc_alb_dif_nir); + auto qc_k = subview_2dkc(d_qc, m_nlay); + auto nc_k = subview_2dkc(d_nc, m_nlay); + auto qi_k = subview_2dkc(d_qi, m_nlay); + auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; + auto rel_k = subview_2dkc(d_rel, m_nlay); + auto rei_k = subview_2dkc(d_rei, m_nlay); + auto sw_flux_up_k = subview_2dk_new(d_sw_flux_up, m_nlay+1); + auto sw_flux_dn_k = subview_2dk_new(d_sw_flux_dn, m_nlay+1); + auto sw_flux_dn_dir_k = subview_2dk_new(d_sw_flux_dn_dir, m_nlay+1); + auto lw_flux_up_k = subview_2dk_new(d_lw_flux_up, m_nlay+1); + auto lw_flux_dn_k = subview_2dk_new(d_lw_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_up_k = subview_2dk_new(d_sw_clnclrsky_flux_up, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = subview_2dk_new(d_sw_clnclrsky_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); + auto sw_clrsky_flux_up_k = subview_2dk_new(d_sw_clrsky_flux_up, m_nlay+1); + auto sw_clrsky_flux_dn_k = subview_2dk_new(d_sw_clrsky_flux_dn, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clrsky_flux_dn_dir, m_nlay+1); + auto sw_clnsky_flux_up_k = subview_2dk_new(d_sw_clnsky_flux_up, m_nlay+1); + auto sw_clnsky_flux_dn_k = subview_2dk_new(d_sw_clnsky_flux_dn, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnsky_flux_dn_dir, m_nlay+1); + auto lw_clnclrsky_flux_up_k = subview_2dk_new(d_lw_clnclrsky_flux_up, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = subview_2dk_new(d_lw_clnclrsky_flux_dn, m_nlay+1); + auto lw_clrsky_flux_up_k = subview_2dk_new(d_lw_clrsky_flux_up, m_nlay+1); + auto lw_clrsky_flux_dn_k = subview_2dk_new(d_lw_clrsky_flux_dn, m_nlay+1); + auto lw_clnsky_flux_up_k = subview_2dk_new(d_lw_clnsky_flux_up, m_nlay+1); + auto lw_clnsky_flux_dn_k = subview_2dk_new(d_lw_clnsky_flux_dn, m_nlay+1); + auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; + auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; + auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; + auto sw_bnd_flux_dif_k = m_buffer.sw_bnd_flux_dif_k; + auto lw_bnd_flux_up_k = m_buffer.lw_bnd_flux_up_k; + auto lw_bnd_flux_dn_k = m_buffer.lw_bnd_flux_dn_k; + auto sfc_flux_dir_vis_k = subview_1dk(d_sfc_flux_dir_vis); + auto sfc_flux_dir_nir_k = subview_1dk(d_sfc_flux_dir_nir); + auto sfc_flux_dif_vis_k = subview_1dk(d_sfc_flux_dif_vis); + auto sfc_flux_dif_nir_k = subview_1dk(d_sfc_flux_dif_nir); + auto aero_tau_sw_k = m_buffer.aero_tau_sw_k; + auto aero_ssa_sw_k = m_buffer.aero_ssa_sw_k; + auto aero_g_sw_k = m_buffer.aero_g_sw_k; + auto aero_tau_lw_k = m_buffer.aero_tau_lw_k; auto cld_tau_sw_bnd_k = subview_3dk(m_buffer.cld_tau_sw_bnd_k); auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); @@ -1015,7 +1068,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif auto d_tint = m_buffer.d_tint; auto d_dz = m_buffer.d_dz; - + auto d_mu0 = m_buffer.cosine_zenith; // Set gas concs to "view" only the first ncol columns #ifdef RRTMGP_ENABLE_YAKL @@ -1032,7 +1085,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // Determine the cosine zenith angle // NOTE: Since we are bridging to F90 arrays this must be done on HOST and then // deep copied to a device view. - auto d_mu0 = m_buffer.cosine_zenith; auto h_mu0 = Kokkos::create_mirror_view(d_mu0); if (m_fixed_solar_zenith_angle > 0) { for (int i=0; i({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1206,7 +1226,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real2d tmp2d = subview_2d(m_buffer.tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real tmp2d_k = subview_2dk(m_buffer.tmp2d_k); + real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1315,7 +1335,7 @@ void RRTMGPRadiation::run_impl (const double dt) { interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), - std::vector({lwp_k, iwp_k})); + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1356,7 +1376,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), - std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1386,9 +1406,9 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS interface_t::rrtmgp_main( ncol, m_nlay, - p_lay_k, t_lay_k, p_lev_k, t_lev_k, + p_lay_k, t_lay_k, p_lev_k, d_tint, m_gas_concs_k, - sfc_alb_dir_k, sfc_alb_dif_k, mu0_k, + sfc_alb_dir_k, sfc_alb_dif_k, d_mu0, lwp_k, iwp_k, rel_k, rei_k, cldfrac_tot_k, aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k, cld_tau_sw_bnd_k, cld_tau_lw_bnd_k, @@ -1412,7 +1432,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, lw_clnsky_flux_up, lw_clnsky_flux_dn}), - std::vector({ + std::vector({ sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, @@ -1420,9 +1440,8 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up_k, lw_clnclrsky_flux_dn_k, lw_clrsky_flux_up_k, lw_clrsky_flux_dn_k, lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); - COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), - std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1472,7 +1491,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), - std::vector({sw_heating_k, lw_heating_k})); + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1516,7 +1535,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), - std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1537,10 +1556,10 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_1d_real cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably @@ -1552,7 +1571,7 @@ void RRTMGPRadiation::run_impl (const double dt) { interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), - std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Compute cloud-top diagnostics following AeroCOM recommendation @@ -1584,17 +1603,17 @@ void RRTMGPRadiation::run_impl (const double dt) { // Get IR 10.5 micron band for COSP auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); - view_1d_real T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); interface_t::compute_aerocom_cloudtop( - ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, + ncol, nlay, t_lay_k, p_lay_k, p_del_k, d_dz, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k); @@ -1602,7 +1621,7 @@ void RRTMGPRadiation::run_impl (const double dt) { T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), - std::vector({ + std::vector({ T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); @@ -1658,34 +1677,8 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; - d_sfc_flux_dir_nir(icol) = sfc_flux_dir_nir_k(i); - d_sfc_flux_dir_vis(icol) = sfc_flux_dir_vis_k(i); - d_sfc_flux_dif_nir(icol) = sfc_flux_dif_nir_k(i); - d_sfc_flux_dif_vis(icol) = sfc_flux_dif_vis_k(i); d_sfc_flux_sw_net(icol) = sw_flux_dn_k(i,kbot_k) - sw_flux_up_k(i,kbot_k); d_sfc_flux_lw_dn(icol) = lw_flux_dn_k(i,kbot_k); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - d_sw_flux_up(icol,k) = sw_flux_up_k(i,k); - d_sw_flux_dn(icol,k) = sw_flux_dn_k(i,k); - d_sw_flux_dn_dir(icol,k) = sw_flux_dn_dir_k(i,k); - d_lw_flux_up(icol,k) = lw_flux_up_k(i,k); - d_lw_flux_dn(icol,k) = lw_flux_dn_k(i,k); - d_sw_clnclrsky_flux_up(icol,k) = sw_clnclrsky_flux_up_k(i,k); - d_sw_clnclrsky_flux_dn(icol,k) = sw_clnclrsky_flux_dn_k(i,k); - d_sw_clnclrsky_flux_dn_dir(icol,k) = sw_clnclrsky_flux_dn_dir_k(i,k); - d_sw_clrsky_flux_up(icol,k) = sw_clrsky_flux_up_k(i,k); - d_sw_clrsky_flux_dn(icol,k) = sw_clrsky_flux_dn_k(i,k); - d_sw_clrsky_flux_dn_dir(icol,k) = sw_clrsky_flux_dn_dir_k(i,k); - d_sw_clnsky_flux_up(icol,k) = sw_clnsky_flux_up_k(i,k); - d_sw_clnsky_flux_dn(icol,k) = sw_clnsky_flux_dn_k(i,k); - d_sw_clnsky_flux_dn_dir(icol,k) = sw_clnsky_flux_dn_dir_k(i,k); - d_lw_clnclrsky_flux_up(icol,k) = lw_clnclrsky_flux_up_k(i,k); - d_lw_clnclrsky_flux_dn(icol,k) = lw_clnclrsky_flux_dn_k(i,k); - d_lw_clrsky_flux_up(icol,k) = lw_clrsky_flux_up_k(i,k); - d_lw_clrsky_flux_dn(icol,k) = lw_clrsky_flux_dn_k(i,k); - d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up_k(i,k); - d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn_k(i,k); - }); // Extract optical properties for COSP Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067_k); @@ -1699,7 +1692,7 @@ void RRTMGPRadiation::run_impl (const double dt) { }); #ifdef RRTMGP_ENABLE_YAKL // Sync back to gas_concs_k - view_3d_real temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); Kokkos::deep_copy(temp, m_gas_concs_k.concs); #endif #endif diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 28c5033c46d4..c60a1cb3181a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -16,17 +16,21 @@ namespace scream { class RRTMGPRadiation : public AtmosphereProcess { public: - using view_1d_real = typename ekat::KokkosTypes::template view_1d; - using view_2d_real = typename ekat::KokkosTypes::template view_2d; - using view_3d_real = typename ekat::KokkosTypes::template view_3d; - using view_2d_real_const = typename ekat::KokkosTypes::template view_2d; - using ci_string = ekat::CaseInsensitiveString; - - using KT = ekat::KokkosTypes; - template - using uview_1d = Unmanaged>; - template - using uview_2d = Unmanaged>; + using KT = ekat::KokkosTypes; + using real1dk = typename KT::template view_1d; + using real2dk = typename KT::template view_2d; + using real3dk = typename KT::template view_3d; + using creal1dk = typename KT::template view_1d; + using creal2dk = typename KT::template view_2d; + using creal3dk = typename KT::template view_3d; + using ureal1dk = Unmanaged; + using ureal2dk = Unmanaged; + using ureal3dk = Unmanaged; + using cureal1dk = Unmanaged; + using cureal2dk = Unmanaged; + using cureal3dk = Unmanaged; + + using ci_string = ekat::CaseInsensitiveString; using layout_t = typename ekat::KokkosTypes::Layout; @@ -91,7 +95,7 @@ class RRTMGPRadiation : public AtmosphereProcess { // These are the gases that we keep track of int m_ngas; std::vector m_gas_names; - view_1d_real m_gas_mol_weights; + real1dk m_gas_mol_weights; #ifdef RRTMGP_ENABLE_YAKL GasConcs m_gas_concs; #endif @@ -128,7 +132,7 @@ class RRTMGPRadiation : public AtmosphereProcess { static constexpr int num_3d_nlay_nlwgpts = 1; // 1d size (ncol) - uview_1d cosine_zenith; + ureal1dk cosine_zenith; #ifdef RRTMGP_ENABLE_YAKL real1d mu0; real1d sfc_alb_dir_vis; @@ -141,19 +145,19 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_1d_real mu0_k; - view_1d_real sfc_alb_dir_vis_k; - view_1d_real sfc_alb_dir_nir_k; - view_1d_real sfc_alb_dif_vis_k; - view_1d_real sfc_alb_dif_nir_k; - view_1d_real sfc_flux_dir_vis_k; - view_1d_real sfc_flux_dir_nir_k; - view_1d_real sfc_flux_dif_vis_k; - view_1d_real sfc_flux_dif_nir_k; + real1dk mu0_k; + real1dk sfc_alb_dir_vis_k; + real1dk sfc_alb_dir_nir_k; + real1dk sfc_alb_dif_vis_k; + real1dk sfc_alb_dif_nir_k; + real1dk sfc_flux_dir_vis_k; + real1dk sfc_flux_dir_nir_k; + real1dk sfc_flux_dif_vis_k; + real1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) - uview_2d d_dz; + ureal2dk d_dz; #ifdef RRTMGP_ENABLE_YAKL real2d p_lay; real2d t_lay; @@ -172,25 +176,25 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real p_lay_k; - view_2d_real t_lay_k; - view_2d_real z_del_k; - view_2d_real p_del_k; - view_2d_real qc_k; - view_2d_real nc_k; - view_2d_real qi_k; - view_2d_real cldfrac_tot_k; - view_2d_real eff_radius_qc_k; - view_2d_real eff_radius_qi_k; - view_2d_real tmp2d_k; - view_2d_real lwp_k; - view_2d_real iwp_k; - view_2d_real sw_heating_k; - view_2d_real lw_heating_k; + real2dk p_lay_k; + real2dk t_lay_k; + real2dk z_del_k; + real2dk p_del_k; + real2dk qc_k; + real2dk nc_k; + real2dk qi_k; + real2dk cldfrac_tot_k; + real2dk eff_radius_qc_k; + real2dk eff_radius_qi_k; + real2dk tmp2d_k; + real2dk lwp_k; + real2dk iwp_k; + real2dk sw_heating_k; + real2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) - uview_2d d_tint; + ureal2dk d_tint; #ifdef RRTMGP_ENABLE_YAKL real2d p_lev; real2d t_lev; @@ -216,28 +220,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real p_lev_k; - view_2d_real t_lev_k; - view_2d_real sw_flux_up_k; - view_2d_real sw_flux_dn_k; - view_2d_real sw_flux_dn_dir_k; - view_2d_real lw_flux_up_k; - view_2d_real lw_flux_dn_k; - view_2d_real sw_clnclrsky_flux_up_k; - view_2d_real sw_clnclrsky_flux_dn_k; - view_2d_real sw_clnclrsky_flux_dn_dir_k; - view_2d_real sw_clrsky_flux_up_k; - view_2d_real sw_clrsky_flux_dn_k; - view_2d_real sw_clrsky_flux_dn_dir_k; - view_2d_real sw_clnsky_flux_up_k; - view_2d_real sw_clnsky_flux_dn_k; - view_2d_real sw_clnsky_flux_dn_dir_k; - view_2d_real lw_clnclrsky_flux_up_k; - view_2d_real lw_clnclrsky_flux_dn_k; - view_2d_real lw_clrsky_flux_up_k; - view_2d_real lw_clrsky_flux_dn_k; - view_2d_real lw_clnsky_flux_up_k; - view_2d_real lw_clnsky_flux_dn_k; + real2dk p_lev_k; + real2dk t_lev_k; + real2dk sw_flux_up_k; + real2dk sw_flux_dn_k; + real2dk sw_flux_dn_dir_k; + real2dk lw_flux_up_k; + real2dk lw_flux_dn_k; + real2dk sw_clnclrsky_flux_up_k; + real2dk sw_clnclrsky_flux_dn_k; + real2dk sw_clnclrsky_flux_dn_dir_k; + real2dk sw_clrsky_flux_up_k; + real2dk sw_clrsky_flux_dn_k; + real2dk sw_clrsky_flux_dn_dir_k; + real2dk sw_clnsky_flux_up_k; + real2dk sw_clnsky_flux_dn_k; + real2dk sw_clnsky_flux_dn_dir_k; + real2dk lw_clnclrsky_flux_up_k; + real2dk lw_clnclrsky_flux_dn_k; + real2dk lw_clrsky_flux_up_k; + real2dk lw_clrsky_flux_dn_k; + real2dk lw_clnsky_flux_up_k; + real2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -248,10 +252,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real sw_bnd_flux_up_k; - view_3d_real sw_bnd_flux_dn_k; - view_3d_real sw_bnd_flux_dir_k; - view_3d_real sw_bnd_flux_dif_k; + real3dk sw_bnd_flux_up_k; + real3dk sw_bnd_flux_dn_k; + real3dk sw_bnd_flux_dir_k; + real3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -260,8 +264,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real lw_bnd_flux_up_k; - view_3d_real lw_bnd_flux_dn_k; + real3dk lw_bnd_flux_up_k; + real3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -270,8 +274,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real sfc_alb_dir_k; - view_2d_real sfc_alb_dif_k; + real2dk sfc_alb_dir_k; + real2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -282,10 +286,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real aero_tau_sw_k; - view_3d_real aero_ssa_sw_k; - view_3d_real aero_g_sw_k; - view_3d_real aero_tau_lw_k; + real3dk aero_tau_sw_k; + real3dk aero_ssa_sw_k; + real3dk aero_g_sw_k; + real3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -294,8 +298,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real cld_tau_sw_bnd_k; - view_3d_real cld_tau_lw_bnd_k; + real3dk cld_tau_sw_bnd_k; + real3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -304,8 +308,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real cld_tau_sw_gpt_k; - view_3d_real cld_tau_lw_gpt_k; + real3dk cld_tau_sw_gpt_k; + real3dk cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index 79f6e570950d..466467919fcf 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -42,14 +42,14 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_test_utils { -using real1dk = typename ekat::KokkosTypes::template view_1d; -using real2dk = typename ekat::KokkosTypes::template view_2d; -using real3dk = typename ekat::KokkosTypes::template view_3d; -using MDRP = typename conv::MDRP; using interface_t = scream::rrtmgp::rrtmgp_interface; +using real1dk = typename interface_t::view_t; +using real2dk = typename interface_t::view_t; +using real3dk = typename interface_t::view_t; +using MDRP = typename conv::MDRP; static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 6e924103229f..8c236e64e811 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -57,9 +57,10 @@ void compute_heating_rate ( View4 const &heating_rate) { using physconst = scream::physics::Constants; + using MDRP = typename conv::MDRP; auto ncol = flux_up.extent(0); auto nlay = flux_up.extent(1)-1; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { heating_rate(icol,ilay) = ( flux_up(icol,ilay+1) - flux_up(icol,ilay) - flux_dn(icol,ilay+1) + flux_dn(icol,ilay) @@ -98,18 +99,20 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } }); auto num_bad = sum(bad_mask); - pass = false; - out << msg << ": " - << num_bad << " values outside range " - << "[" << xmin << "," << xmax << "]" - << "; minval = " << _xmin - << "; maxval = " << _xmax << "\n"; + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } } return pass; } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template ::type* = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; @@ -117,10 +120,10 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - Kokkos::View bad_mask("bad_mask", x.size()); - Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { - if (x.data()[i] < xmin or x.data()[i] > xmax) { - bad_mask.data()[i] = true; + Kokkos::View bad_mask("bad_mask", x.extent(0)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + if (x(i) < xmin or x(i) > xmax) { + bad_mask(i) = true; } }); auto num_bad = conv::sum(bad_mask); @@ -133,6 +136,68 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val } return pass; } + +template ::type* = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + if (x(i, j) < xmin or x(i, j) > xmax) { + bad_mask(i, j) = true; + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + +template ::type* = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1), x.extent(2)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + for (size_t k = 0; k < x.extent(2); ++k) { + if (x(i, j, k) < xmin or x(i, j, k) > xmax) { + bad_mask(i, j, k) = true; + } + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + + #endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 2038cb62ff79..c82d9e38a3b6 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -159,7 +159,7 @@ int get_wavelength_index_lw(double wavelength); // New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_interface { using MDRP = typename conv::MDRP; @@ -980,7 +980,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i // Kokkos::deep_copy(seeds_host, seeds); // for (int icol = 0; icol < ncol; ++icol) { // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // Kokkos::parallel_for(MDRP::template get<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { // auto generator = random_pool.get_state(); // cldx(icol,ilay,igpt) = generator.drand(0., 1.); // random_pool.free_state(generator); @@ -996,7 +996,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i }); // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { for (int ilay = 1; ilay < nlay; ilay++) { // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { @@ -1016,7 +1016,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i } // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { subcolumn_mask(icol,ilay,igpt) = 1; } else { @@ -1202,13 +1202,21 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template +template::type* = nullptr> static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + Kokkos::parallel_for(arr_out.size(), KOKKOS_LAMBDA(int i) { + arr_out(i) = std::min(std::max(arr_in(i), lower), upper); + }); +} + +template::type* = nullptr> +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { + Kokkos::parallel_for(MDRP::template get<2>({arr_out.extent(0), arr_out.extent(1)}), KOKKOS_LAMBDA(int i, int j) { + arr_out(i, j) = std::min(std::max(arr_in(i, j), lower), upper); }); } + static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) { // Get wavelength bounds for all wavelength bands @@ -1382,7 +1390,7 @@ static OpticalProps1sclK get_subsampled_clouds( auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { auto ibnd = gpoint_bands(igpt); if (cldmask(icol,ilay,igpt) == 1) { subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 65d1da9f1d68..0d3f18e7d841 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -320,6 +320,12 @@ int run_yakl(int argc, char** argv) { int run_kokkos(int argc, char** argv) { using namespace ekat::logger; using logger_t = Logger; + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using MDRP = utils_t::MDRP; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; ekat::Comm comm(MPI_COMM_WORLD); auto logger = std::make_shared("",LogLevel::info,comm); @@ -374,7 +380,7 @@ int run_kokkos(int argc, char** argv) { real2dk sw_flux_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Get dimension sizes int ncol = sw_flux_up_ref.extent(0); @@ -394,12 +400,12 @@ int run_kokkos(int argc, char** argv) { real2dk p_lev("p_lev", ncol, nlay+1); real2dk t_lev("t_lev", ncol, nlay+1); real2dk col_dry; - GasConcsK gas_concs; + GasConcsK gas_concs; read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry, ncol); // Initialize absorption coefficients logger->info("Initialize RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Setup our dummy atmosphere based on the input data we read in logger->info("Setup dummy atmos...\n"); @@ -413,7 +419,7 @@ int run_kokkos(int argc, char** argv) { real2dk rel("rel", ncol, nlay); real2dk rei("rei", ncol, nlay); real2dk cld("cld", ncol, nlay); - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol, p_lay, t_lay, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -426,8 +432,8 @@ int run_kokkos(int argc, char** argv) { // we would just have to setup the pointers to them in the // FluxesBroadband object logger->info("Setup fluxes...\n"); - const auto nswbands = scream::rrtmgp::k_dist_sw_k.get_nband(); - const auto nlwbands = scream::rrtmgp::k_dist_lw_k.get_nband(); + const auto nswbands = interface_t::k_dist_sw_k.get_nband(); + const auto nlwbands = interface_t::k_dist_lw_k.get_nband(); real2dk sw_flux_up ("sw_flux_up" , ncol, nlay+1); real2dk sw_flux_dn ("sw_flux_dn" , ncol, nlay+1); real2dk sw_flux_dir("sw_flux_dir", ncol, nlay+1); @@ -457,7 +463,7 @@ int run_kokkos(int argc, char** argv) { // Compute band-by-band surface_albedos. real2dk sfc_alb_dir("sfc_alb_dir", ncol, nswbands); real2dk sfc_alb_dif("sfc_alb_dif", ncol, nswbands); - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -468,19 +474,19 @@ int run_kokkos(int argc, char** argv) { auto aer_ssa_sw = real3dk("aer_ssa_sw", ncol, nlay, nswbands); auto aer_asm_sw = real3dk("aer_asm_sw", ncol, nlay, nswbands); auto aer_tau_lw = real3dk("aer_tau_lw", ncol, nlay, nlwbands); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_sw(icol,ilay,ibnd) = 0; aer_ssa_sw(icol,ilay,ibnd) = 0; aer_asm_sw(icol,ilay,ibnd) = 0; }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_lw(icol,ilay,ibnd) = 0; }); // These are returned as outputs now from rrtmgp_main // TODO: provide as inputs consistent with how aerosol is treated? - const auto nswgpts = scream::rrtmgp::k_dist_sw_k.get_ngpt(); - const auto nlwgpts = scream::rrtmgp::k_dist_lw_k.get_ngpt(); + const auto nswgpts = interface_t::k_dist_sw_k.get_ngpt(); + const auto nlwgpts = interface_t::k_dist_lw_k.get_ngpt(); auto cld_tau_sw_bnd = real3dk("cld_tau_sw_bnd", ncol, nlay, nswbands); auto cld_tau_lw_bnd = real3dk("cld_tau_lw_bnd", ncol, nlay, nlwbands); auto cld_tau_sw = real3dk("cld_tau_sw", ncol, nlay, nswgpts); @@ -489,7 +495,7 @@ int run_kokkos(int argc, char** argv) { // Run RRTMGP code on dummy atmosphere logger->info("Run RRTMGP...\n"); const Real tsi_scaling = 1; - scream::rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, nlay, p_lay, t_lay, p_lev, t_lev, gas_concs, sfc_alb_dir, sfc_alb_dif, mu0, @@ -513,33 +519,33 @@ int run_kokkos(int argc, char** argv) { // Check values against baseline logger->info("Check values...\n"); - rrtmgpTest::read_fluxes( + utils_t::read_fluxes( baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; // Because the aerosol optical properties are all set to zero, these fluxes must be equal - if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; logger->info("Cleaning up...\n"); // Clean up or else YAKL will throw errors - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); scream::finalize_kls(); return nerr != 0 ? 1 : 0; diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 36e2a59e02ff..99a582445616 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -856,6 +856,7 @@ using real3dk = interface_t::view_t; using int1dk = interface_t::view_t; using int2dk = interface_t::view_t; using int3dk = interface_t::view_t; +using MDRP = interface_t::MDRP; TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos @@ -1082,7 +1083,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Need to initialize RRTMGP with dummy gases logger->info("Init gases...\n"); - GasConcsK gas_concs; + GasConcsK gas_concs; string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); @@ -1100,7 +1101,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { auto sw_bnd_flux_dir = real3dk("sw_bnd_flux_dir", ncol, nlay+1, nbnd); auto sw_bnd_flux_dif = real3dk("sw_bnd_flux_dif", ncol, nlay+1, nbnd); logger->info("Populate band-resolved 3d fluxes for test case with only transition band flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1132,7 +1133,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in NIR bands logger->info("Populate band-resolved 3d fluxes for test case with only NIR flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1; sw_bnd_flux_dif(icol,ilay,ibnd) = 1; @@ -1163,7 +1164,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in VIS bands logger->info("Populate band-resolved 3d fluxes for test case with only VIS/UV flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1194,7 +1195,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in all bands logger->info("Populate band-resolved 3d fluxes for test with non-zero flux in all bands...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1.0; sw_bnd_flux_dif(icol,ilay,ibnd) = 2.0; @@ -1293,13 +1294,13 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { for (int igpt = 0; igpt < ngpt; ++igpt) { real cldmask_real = cldmask(icol,ilay,igpt); cldfrac_from_mask(icol,ilay) += cldmask_real; } }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { cldfrac_from_mask(icol,ilay) = cldfrac_from_mask(icol,ilay) / ngpt; }); // For cldfrac 1 we should get 1, for cldfrac 0 we should get 0, but in between we cannot be sure diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 9e6e94a2164f..75feeb1b8dd4 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -422,6 +422,13 @@ TEST_CASE("rrtmgp_scream_standalone", "") { #endif #ifdef RRTMGP_ENABLE_KOKKOS TEST_CASE("rrtmgp_scream_standalone_k", "") { + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using MDRP = interface_t::MDRP; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; + // Get baseline name (needs to be passed as an arg) std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); @@ -439,7 +446,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { real2dk sw_flux_dn_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Load ad parameter list std::string fname = "input_unit.yaml"; @@ -526,11 +533,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { // this will copy the first column of the input data (the first profile) ncol // times. We will then fill some fraction of these columns with clouds for // the test problem. - GasConcsK gas_concs; + GasConcsK gas_concs; real2dk col_dry; read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, col_dry, ncol_all); // Setup dummy problem; need to use tmp arrays with ncol_all size - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol_all, p_lay_all, t_lay_all, sfc_alb_dir_vis_all, sfc_alb_dir_nir_all, sfc_alb_dif_vis_all, sfc_alb_dif_nir_all, @@ -547,7 +554,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { sfc_alb_dif_nir(icol) = sfc_alb_dif_nir_all(icol_all); mu0(icol) = mu0_all(icol_all); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lay(icol,ilay) = p_lay_all(icol_all,ilay); t_lay(icol,ilay) = t_lay_all(icol_all,ilay); @@ -557,14 +564,14 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { rei(icol,ilay) = rei_all(icol_all,ilay); cld(icol,ilay) = cld_all(icol_all,ilay); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lev(icol,ilay) = p_lev_all(icol_all,ilay); t_lev(icol,ilay) = t_lev_all(icol_all,ilay); }); // Need to calculate a dummy pseudo_density for our test problem - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { p_del(icol,ilay) = abs(p_lev(icol,ilay+1) - p_lev(icol,ilay)); }); @@ -573,13 +580,13 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto qc = real2dk("qc", ncol, nlay); auto nc = real2dk("nc", ncol, nlay); auto qi = real2dk("qi", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); qi(icol,ilay) = 1e-3 * iwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); }); // Copy gases from gas_concs to gas_vmr array - Kokkos::parallel_for(conv::get_mdrp<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { + Kokkos::parallel_for(MDRP::template get<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { auto icol_all = ncol * irank + icol; gas_vmr(icol,ilay,igas) = gas_concs.concs(icol_all,ilay,igas); }); @@ -691,7 +698,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { const int i = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - if (k < nlay) t_lay(i+1,k+1) = d_tmid(i,k); + if (k < nlay) t_lay(i,k) = d_tmid(i,k); sw_flux_up_test(i,k) = d_sw_flux_up(i,k); sw_flux_dn_test(i,k) = d_sw_flux_dn(i,k); @@ -722,7 +729,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto sw_flux_dn_dir_loc = real2dk("sw_flux_dn_dir_loc", ncol, nlay+1); auto lw_flux_up_loc = real2dk("lw_flux_up_loc" , ncol, nlay+1); auto lw_flux_dn_loc = real2dk("lw_flux_dn_loc" , ncol, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; sw_flux_up_loc(icol,ilay) = sw_flux_up_ref(icol_all,ilay); sw_flux_dn_loc(icol,ilay) = sw_flux_dn_ref(icol_all,ilay); @@ -730,11 +737,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { lw_flux_up_loc(icol,ilay) = lw_flux_up_ref(icol_all,ilay); lw_flux_dn_loc(icol,ilay) = lw_flux_dn_ref(icol_all,ilay); }); - REQUIRE(rrtmgpTest::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); + REQUIRE(utils_t::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); // Finalize the driver. YAKL will be finalized inside // RRTMGPRadiation::finalize_impl after RRTMGP has had the From 4badc7bbad04be736fd7f046cfc526f275d46f21 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 14:50:19 -0600 Subject: [PATCH 031/477] cleanup --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 89 ++++++------------- 1 file changed, 27 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1556280eab4b..cde298cc8cd9 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -19,28 +19,6 @@ namespace scream { -template -void check_views2(const View1& view1, const View2& view2) -{ - constexpr auto krank1 = View1::rank; - constexpr auto krank2 = View2::rank; - - RRT_REQUIRE(krank1 == krank2, "Rank mismatch for: " << view1.label()); - RRT_REQUIRE(krank1 == 2, "Rank mismatch for: " << view1.label()); - - for (auto r = 0; r < krank1; ++r) { - RRT_REQUIRE(view1.extent(r) == view2.extent(r), "Dim mismatch for: " << view1.label() << ", rank: " << r << ", " << view1.extent(r) << " != " << view2.extent(r)); - } - - for (auto i = 0; i < view1.extent(0); ++i) { - for (auto j = 0; j < view1.extent(1); ++j) { - const auto data1 = view1(i, j); - const auto data2 = view2(i, j); - RRT_REQUIRE(data1 == data2, "Data mismatch for: " << view1.label() << ", i: " << i << ", j: " << j << ", " << data1 << " != " << data2); - } - } -} - using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; @@ -967,8 +945,8 @@ void RRTMGPRadiation::run_impl (const double dt) { return subv; #endif }; - auto subview_2dk = [&](const ureal2dk& v) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); #ifdef RRTMGP_ENABLE_YAKL real2dk rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); @@ -985,16 +963,6 @@ void RRTMGPRadiation::run_impl (const double dt) { return rv; #else return subv; -#endif - }; - auto subview_2dk_new = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); -#ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; #endif }; auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { @@ -1027,26 +995,26 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; auto rel_k = subview_2dkc(d_rel, m_nlay); auto rei_k = subview_2dkc(d_rei, m_nlay); - auto sw_flux_up_k = subview_2dk_new(d_sw_flux_up, m_nlay+1); - auto sw_flux_dn_k = subview_2dk_new(d_sw_flux_dn, m_nlay+1); - auto sw_flux_dn_dir_k = subview_2dk_new(d_sw_flux_dn_dir, m_nlay+1); - auto lw_flux_up_k = subview_2dk_new(d_lw_flux_up, m_nlay+1); - auto lw_flux_dn_k = subview_2dk_new(d_lw_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_up_k = subview_2dk_new(d_sw_clnclrsky_flux_up, m_nlay+1); - auto sw_clnclrsky_flux_dn_k = subview_2dk_new(d_sw_clnclrsky_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); - auto sw_clrsky_flux_up_k = subview_2dk_new(d_sw_clrsky_flux_up, m_nlay+1); - auto sw_clrsky_flux_dn_k = subview_2dk_new(d_sw_clrsky_flux_dn, m_nlay+1); - auto sw_clrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clrsky_flux_dn_dir, m_nlay+1); - auto sw_clnsky_flux_up_k = subview_2dk_new(d_sw_clnsky_flux_up, m_nlay+1); - auto sw_clnsky_flux_dn_k = subview_2dk_new(d_sw_clnsky_flux_dn, m_nlay+1); - auto sw_clnsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnsky_flux_dn_dir, m_nlay+1); - auto lw_clnclrsky_flux_up_k = subview_2dk_new(d_lw_clnclrsky_flux_up, m_nlay+1); - auto lw_clnclrsky_flux_dn_k = subview_2dk_new(d_lw_clnclrsky_flux_dn, m_nlay+1); - auto lw_clrsky_flux_up_k = subview_2dk_new(d_lw_clrsky_flux_up, m_nlay+1); - auto lw_clrsky_flux_dn_k = subview_2dk_new(d_lw_clrsky_flux_dn, m_nlay+1); - auto lw_clnsky_flux_up_k = subview_2dk_new(d_lw_clnsky_flux_up, m_nlay+1); - auto lw_clnsky_flux_dn_k = subview_2dk_new(d_lw_clnsky_flux_dn, m_nlay+1); + auto sw_flux_up_k = subview_2dk(d_sw_flux_up, m_nlay+1); + auto sw_flux_dn_k = subview_2dk(d_sw_flux_dn, m_nlay+1); + auto sw_flux_dn_dir_k = subview_2dk(d_sw_flux_dn_dir, m_nlay+1); + auto lw_flux_up_k = subview_2dk(d_lw_flux_up, m_nlay+1); + auto lw_flux_dn_k = subview_2dk(d_lw_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_up_k = subview_2dk(d_sw_clnclrsky_flux_up, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = subview_2dk(d_sw_clnclrsky_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); + auto sw_clrsky_flux_up_k = subview_2dk(d_sw_clrsky_flux_up, m_nlay+1); + auto sw_clrsky_flux_dn_k = subview_2dk(d_sw_clrsky_flux_dn, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = subview_2dk(d_sw_clrsky_flux_dn_dir, m_nlay+1); + auto sw_clnsky_flux_up_k = subview_2dk(d_sw_clnsky_flux_up, m_nlay+1); + auto sw_clnsky_flux_dn_k = subview_2dk(d_sw_clnsky_flux_dn, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = subview_2dk(d_sw_clnsky_flux_dn_dir, m_nlay+1); + auto lw_clnclrsky_flux_up_k = subview_2dk(d_lw_clnclrsky_flux_up, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = subview_2dk(d_lw_clnclrsky_flux_dn, m_nlay+1); + auto lw_clrsky_flux_up_k = subview_2dk(d_lw_clrsky_flux_up, m_nlay+1); + auto lw_clrsky_flux_dn_k = subview_2dk(d_lw_clrsky_flux_dn, m_nlay+1); + auto lw_clnsky_flux_up_k = subview_2dk(d_lw_clnsky_flux_up, m_nlay+1); + auto lw_clnsky_flux_dn_k = subview_2dk(d_lw_clnsky_flux_dn, m_nlay+1); auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; @@ -1224,9 +1192,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // correct size that uses m_buffer.tmp2d's pointer #ifdef RRTMGP_ENABLE_YAKL real2d tmp2d = subview_2d(m_buffer.tmp2d); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1235,22 +1200,22 @@ void RRTMGPRadiation::run_impl (const double dt) { // 'o3' is marked as 'Required' rather than 'Computed', so we need to get the proper field auto f = name=="o3" ? get_field_in(full_name) : get_field_out(full_name); auto d_vmr = f.get_view(); +#ifdef RRTMGP_ENABLE_KOKKOS + auto tmp2d_k = subview_2dkc(d_vmr, m_nlay); +#endif +#ifdef RRTMGP_ENABLE_YAKL // Copy to YAKL const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { -#ifdef RRTMGP_ENABLE_YAKL tmp2d(i+1,k+1) = d_vmr(icol,k); // Note that for YAKL arrays i and k start with index 1 -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - tmp2d_k(i,k) = d_vmr(icol,k); -#endif }); }); Kokkos::fence(); +#endif // Populate GasConcs object #ifdef RRTMGP_ENABLE_YAKL From 46595e4404340126f43f126df89776a563ad140a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 15:48:32 -0600 Subject: [PATCH 032/477] subview_2dk bug fix --- .../eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index cde298cc8cd9..9fab85444fcc 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -946,7 +946,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); + ureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); #ifdef RRTMGP_ENABLE_YAKL real2dk rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); From a11d4e51b372ba2b27bd8130a99d8af021e1ee86 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 17:44:50 -0600 Subject: [PATCH 033/477] More fixes --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 9fab85444fcc..7a5af5843d19 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -926,7 +926,7 @@ void RRTMGPRadiation::run_impl (const double dt) { // check_range_k will fail due to looking at unused values. Once rrtmgp can handle // packs, this won't be necessary. auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { - ureal1dk subv(v, std::make_pair(0, ncol)); + ureal1dk subv(v, std::make_pair(beg, beg+ncol)); #ifdef RRTMGP_ENABLE_YAKL real1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); @@ -936,7 +936,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { - cureal1dk subv(v, std::make_pair(0, ncol)); + cureal1dk subv(v, std::make_pair(beg, beg+ncol)); #ifdef RRTMGP_ENABLE_YAKL creal1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); @@ -966,7 +966,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { - ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); // The range assumes these are buffer views, not fields #ifdef RRTMGP_ENABLE_YAKL real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); From 325c3a3f8f29b97e6246d94fe2c9b88308571813 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 13:04:44 -0600 Subject: [PATCH 034/477] Set to correct rrtmgp branch --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 4c542032b3ac..02c2e03df33e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 4c542032b3ac3f842a9619c9eadc005c507a0366 +Subproject commit 02c2e03df33edca6101670168ca187cb94c0768e From aa300be9f1ae3fc732b2228464d07ab390311e86 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 14:01:18 -0600 Subject: [PATCH 035/477] Add buffer views should be unmanaged --- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index c60a1cb3181a..5dece435fb0a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -145,15 +145,15 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + ureal1dk mu0_k; + ureal1dk sfc_alb_dir_vis_k; + ureal1dk sfc_alb_dir_nir_k; + ureal1dk sfc_alb_dif_vis_k; + ureal1dk sfc_alb_dif_nir_k; + ureal1dk sfc_flux_dir_vis_k; + ureal1dk sfc_flux_dir_nir_k; + ureal1dk sfc_flux_dif_vis_k; + ureal1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) @@ -176,21 +176,21 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + ureal2dk p_lay_k; + ureal2dk t_lay_k; + ureal2dk z_del_k; + ureal2dk p_del_k; + ureal2dk qc_k; + ureal2dk nc_k; + ureal2dk qi_k; + ureal2dk cldfrac_tot_k; + ureal2dk eff_radius_qc_k; + ureal2dk eff_radius_qi_k; + ureal2dk tmp2d_k; + ureal2dk lwp_k; + ureal2dk iwp_k; + ureal2dk sw_heating_k; + ureal2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) @@ -220,28 +220,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + ureal2dk p_lev_k; + ureal2dk t_lev_k; + ureal2dk sw_flux_up_k; + ureal2dk sw_flux_dn_k; + ureal2dk sw_flux_dn_dir_k; + ureal2dk lw_flux_up_k; + ureal2dk lw_flux_dn_k; + ureal2dk sw_clnclrsky_flux_up_k; + ureal2dk sw_clnclrsky_flux_dn_k; + ureal2dk sw_clnclrsky_flux_dn_dir_k; + ureal2dk sw_clrsky_flux_up_k; + ureal2dk sw_clrsky_flux_dn_k; + ureal2dk sw_clrsky_flux_dn_dir_k; + ureal2dk sw_clnsky_flux_up_k; + ureal2dk sw_clnsky_flux_dn_k; + ureal2dk sw_clnsky_flux_dn_dir_k; + ureal2dk lw_clnclrsky_flux_up_k; + ureal2dk lw_clnclrsky_flux_dn_k; + ureal2dk lw_clrsky_flux_up_k; + ureal2dk lw_clrsky_flux_dn_k; + ureal2dk lw_clnsky_flux_up_k; + ureal2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -252,10 +252,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + ureal3dk sw_bnd_flux_up_k; + ureal3dk sw_bnd_flux_dn_k; + ureal3dk sw_bnd_flux_dir_k; + ureal3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -264,8 +264,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + ureal3dk lw_bnd_flux_up_k; + ureal3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -274,8 +274,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + ureal2dk sfc_alb_dir_k; + ureal2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -286,10 +286,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + ureal3dk aero_tau_sw_k; + ureal3dk aero_ssa_sw_k; + ureal3dk aero_g_sw_k; + ureal3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -298,8 +298,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + ureal3dk cld_tau_sw_bnd_k; + ureal3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -308,8 +308,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + ureal3dk cld_tau_sw_gpt_k; + ureal3dk cld_tau_lw_gpt_k; #endif }; From 75ae15d16933b68f9343c0d694fc884622cbc7c0 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 15:55:26 -0600 Subject: [PATCH 036/477] Build fixes for yakl mode --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 4 ++-- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 7a5af5843d19..62de7f8416cd 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -1293,8 +1293,8 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - interface_t::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - interface_t::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 5dece435fb0a..e036e7ac08c0 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -34,7 +34,9 @@ class RRTMGPRadiation : public AtmosphereProcess { using layout_t = typename ekat::KokkosTypes::Layout; +#ifdef RRTMGP_ENABLE_KOKKOS using interface_t = rrtmgp::rrtmgp_interface; +#endif // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); From 8603487991499be6db1ce149a496333084c28dd9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 10:45:31 -0600 Subject: [PATCH 037/477] EAMxx: fix how we pass IOType to scorpio The enum underlying values are not to be trusted. Do a manual conversion to PIO_IOTYPE instead. --- .../src/share/io/scream_scorpio_interface.cpp | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index f354b44efc1d..cb2b501f2b5c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -207,6 +207,21 @@ size_t dtype_size (const std::string& dtype) { return 0; } +int pio_iotype (IOType iotype) { + int iotype_int; + auto& s = ScorpioSession::instance(); + switch(iotype){ + case IOType::DefaultIOType: iotype_int = s.pio_type_default; break; + case IOType::NetCDF: iotype_int = static_cast(PIO_IOTYPE_NETCDF); break; + case IOType::PnetCDF: iotype_int = static_cast(PIO_IOTYPE_PNETCDF); break; + case IOType::Adios: iotype_int = static_cast(PIO_IOTYPE_ADIOS); break; + case IOType::Hdf5: iotype_int = static_cast(PIO_IOTYPE_HDF5); break; + default: + EKAT_ERROR_MSG ("Unrecognized/unsupported iotype.\n"); + } + return iotype_int; +} + // ====================== Local utilities ========================== // namespace impl { @@ -313,9 +328,9 @@ void init_subsystem(const ekat::Comm& comm, const int atm_id) s.pio_rearranger = PIO_REARR_SUBSET; s.pio_format = PIO_64BIT_DATA; #if PIO_USE_PNETCDF - s.pio_type_default = PIO_IOTYPE_PNETCDF; + s.pio_type_default = pio_iotype(IOType::PnetCDF); #elif PIO_USE_NETCDF - s.pio_type_default = PIO_IOTYPE_NETCDF; + s.pio_type_default = pio_iotype(IOType::NetCDF); #else #error "Standalone EAMxx requires either PNETCDF or NETCDF iotype to be available in Scorpio" #endif @@ -395,9 +410,7 @@ void register_file (const std::string& filename, if (f.mode == Unset) { // First time we ask for this file. Call PIO open routine(s) int err; - int iotype_int = iotype==IOType::DefaultIOType - ? s.pio_type_default - : static_cast(iotype); + int iotype_int = pio_iotype(iotype); if (mode & Read) { auto write = mode & Write ? PIO_WRITE : PIO_NOWRITE; err = PIOc_openfile(s.pio_sysid,&f.ncid,&iotype_int,filename.c_str(),write); From 7ecf8709c508467914314b5a85af737420e44dbd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 14:03:06 -0600 Subject: [PATCH 038/477] PyEAMxx: changed how user creates grids - Make user create a GridsManager rather than a single grid, to prepare for more complex use cases (e.g., dyn-phys) - Create a PySession singleton to store some data - Misc fixes to p3_standalone_py test --- components/eamxx/src/python/CMakeLists.txt | 9 ++- components/eamxx/src/python/pyatmproc.hpp | 41 ++++++----- components/eamxx/src/python/pyeamxx.cpp | 8 +++ components/eamxx/src/python/pyeamxx.hpp | 30 ++++++++ components/eamxx/src/python/pygrid.hpp | 72 +++++++------------ .../eamxx/tests/python/pyp3/p3_standalone_py | 22 +++--- 6 files changed, 106 insertions(+), 76 deletions(-) create mode 100644 components/eamxx/src/python/pyeamxx.hpp diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 8c7cf2aaba9c..43e2db73e533 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -2,4 +2,11 @@ find_package(pybind11 REQUIRED) find_package(mpi4py REQUIRED) pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC mpi4py scream_share scream_io diagnostics eamxx_physics) +target_link_libraries(pyeamxx PUBLIC + mpi4py + scream_share + scream_io + diagnostics + eamxx_physics + scream_test_support +) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index 4e7a88b21e67..f4daf87233ff 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -10,6 +10,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" +#include "pyeamxx.hpp" #include @@ -20,7 +21,6 @@ namespace scream { struct PyAtmProc { std::shared_ptr ap; - PyGrid phys_grid; std::map fields; util::TimeStamp t0; util::TimeStamp time; @@ -28,14 +28,10 @@ struct PyAtmProc { std::shared_ptr output_mgr; - PyAtmProc (const PyParamList& params, const PyGrid& phys_grid_in) - : phys_grid(phys_grid_in) + PyAtmProc (const PyParamList& params) { // Get the comm - const auto& comm = phys_grid.grid->get_comm(); - - // Create a grids manager on the fly - auto gm = std::make_shared(phys_grid.grid); + const auto& comm = PySession::get().comm; // Create the atm proc register_physics(); @@ -46,6 +42,7 @@ struct PyAtmProc { ap = apf.create(ap_type,comm,params.pl); // Create the fields + auto gm = PySession::get().gm; ap->set_grids(gm); create_fields(); } @@ -118,33 +115,41 @@ struct PyAtmProc { } } } - AtmosphereInput reader (ic_filename,phys_grid.grid,ic_fields,true); - reader.read_variables(); + if (ic_fields.size()>0) { + const auto& gn = ic_fields[0].get_header().get_identifier().get_grid_name(); + auto gm = PySession::get().gm; + auto grid = gm->get_grid(gn); + AtmosphereInput reader (ic_filename,grid,ic_fields,true); + reader.read_variables(); + } scorpio::release_file(ic_filename); return pybind11::cast(missing); } void setup_output (const std::string& yaml_file) { + auto comm = PySession::get().comm; // Load output params auto params = ekat::parse_yaml_file(yaml_file); // Stuff all fields in a field manager - auto fm = std::make_shared(phys_grid.grid); - fm->registration_begins(); - fm->registration_ends(); + auto gm = PySession::get().gm; + std::map> fms; + for (auto it : gm->get_repo()) { + fms[it.first] = std::make_shared(it.second); + fms[it.first]->registration_begins(); + fms[it.first]->registration_ends(); + } for (auto it : fields) { - fm->add_field(it.second.f); + const auto& gn = it.second.f.get_header().get_identifier().get_grid_name(); + fms.at(gn)->add_field(it.second.f); } - // Create a grids manager on the fly - auto gm = std::make_shared(phys_grid.grid); - // Make sure diagnostics are available, then create the output mgr register_diagnostics(); output_mgr = std::make_shared(); - output_mgr->setup(phys_grid.grid->get_comm(),params,fm,gm,t0,t0,false); + output_mgr->setup(comm,params,fms,gm,t0,t0,false); output_mgr->set_logger(ap->get_logger()); } @@ -161,7 +166,7 @@ struct PyAtmProc { inline void pybind_pyatmproc(pybind11::module& m) { pybind11::class_(m,"AtmProc") - .def(pybind11::init()) + .def(pybind11::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) .def("setup_output",&PyAtmProc::setup_output) diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index 5834ca10f74f..93a1e0ab2dfa 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -19,6 +19,10 @@ void initialize (MPI_Comm mpi_comm) { ekat::Comm comm(mpi_comm); initialize_scream_session(comm.am_i_root()); scorpio::init_subsystem(comm); + + auto& s = PySession::get(); + s.comm = comm; + s.inited = true; } void initialize () { @@ -28,6 +32,10 @@ void initialize (pybind11::object py_comm) { initialize(get_c_comm(py_comm)); } void finalize () { + auto& s = PySession::get(); + s.gm = nullptr; + s.inited = false; + scorpio::finalize_subsystem(); finalize_scream_session(); } diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/pyeamxx.hpp new file mode 100644 index 000000000000..59371844a763 --- /dev/null +++ b/components/eamxx/src/python/pyeamxx.hpp @@ -0,0 +1,30 @@ +#ifndef PYEAMXX_HPP +#define PYEAMXX_HPP + +#include "physics/register_physics.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "dynamics/register_dynamics.hpp" + +#include "share/grid/grids_manager.hpp" + +#include + +namespace scream { + +struct PySession { + static PySession& get () { + static PySession s; + return s; + } + + ekat::Comm comm; + std::shared_ptr gm; + bool inited = false; +private: + + PySession () = default; +}; + +} // namespace scream + +#endif // PYEAMXX_HPP diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp index 874b1aef17a0..102cf6037c62 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/pygrid.hpp @@ -1,10 +1,9 @@ #ifndef PYGRID_HPP #define PYGRID_HPP -#include "share/grid/grids_manager.hpp" -#include "share/grid/point_grid.hpp" -#include "pyfield.hpp" -#include "pyutils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "pyeamxx.hpp" #include @@ -12,53 +11,34 @@ namespace scream { -// Small grids manager class, to hold a pre-built grid -// We will use this to build a GM on the fly from a single grid -class SingleGridGM : public GridsManager +inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) { -public: - SingleGridGM (const std::shared_ptr& grid) - { - add_grid(grid); + EKAT_REQUIRE_MSG (PySession::get().inited, + "Error! You did not initialize pyeamxx, or you already finalized it!\n"); + auto& comm = PySession::get().comm; + ekat::ParameterList gm_params; + std::vector grids_names = {"Physics"}; + auto& pl = gm_params.sublist("Physics"); + pl.set("type",std::string("point_grid")); + pl.set("number_of_global_columns",ncols); + pl.set("number_of_vertical_levels",nlevs); + gm_params.set("grids_names",grids_names); + + if (latlon_nc_file!="") { + gm_params.set("ic_filename",latlon_nc_file); } - std::string name () const override { return "SingleGridGM"; } - - void build_grids () override {} - -protected: - remapper_ptr_type - do_create_remapper (const grid_ptr_type /* from_grid */, - const grid_ptr_type /* to_grid */) const override - { - EKAT_ERROR_MSG ("Error! do_create_remapper not implemented for SingleGridGM.\n"); - } -}; - -struct PyGrid { - std::shared_ptr grid; - - PyGrid () = default; - - PyGrid(const std::string& name, int ncols, int nlevs) - : PyGrid (name,ncols,nlevs,MPI_COMM_WORLD) - {} - - PyGrid(const std::string& name, int ncols, int nlevs, pybind11::object py_comm) - : PyGrid (name,ncols,nlevs,get_c_comm(py_comm)) - {} - - PyGrid(const std::string& name, int ncols, int nlevs, MPI_Comm mpi_comm) - { - ekat::Comm comm(mpi_comm); - grid = create_point_grid(name,ncols,nlevs,comm); - } -}; + PySession::get().gm = create_mesh_free_grids_manager (comm, gm_params); + PySession::get().gm->build_grids(); +} +inline void create_grids_manager (int ncols, int nlevs) +{ + create_grids_manager(ncols,nlevs,""); +} inline void pybind_pygrid (pybind11::module& m) { - pybind11::class_(m,"Grid") - .def(pybind11::init()) - .def(pybind11::init()); + m.def("create_grids_manager",pybind11::overload_cast(&create_grids_manager)); + m.def("create_grids_manager",pybind11::overload_cast(&create_grids_manager)); } } // namespace scream diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index 9d2b648060bf..def3c7f01b6d 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -6,7 +6,7 @@ import sys sys.path.append('@SCREAM_BASE_DIR@/scripts') # Add path to pyeamxx libs -sys.path.append('@CMAKE_BINARY_DIR@/src/python/libpyeamxx') +sys.path.append('@CMAKE_BINARY_DIR@/src/python') # Without these, and manual init/finalize, on my laptop I get # obscure MPI errors at exit. @@ -15,8 +15,7 @@ mpi4py.rc.initialize = False # do not initialize MPI automatically mpi4py.rc.finalize = False # do not finalize MPI automatically from mpi4py import MPI -import pyscream as ps -import pyp3 +import pyeamxx from pathlib import Path from utils import ensure_yaml @@ -34,17 +33,18 @@ def main (): dt = yaml_input['time_stepping']['time_step'] t0_str = yaml_input['time_stepping']['run_t0'] + ic_file = Path('@SCREAM_DATA_DIR@/init/screami_unit_tests_ne2np4L72_20220822.nc') + # Create the grid ncols = 218 nlevs = 72 - grid = ps.Grid("Physics",ncols,nlevs) - - ic_file = Path('@SCREAM_DATA_DIR@/init/screami_unit_tests_ne2np4L72_20220822.nc') + pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) - p3params = ps.ParameterList(yaml_input['atmosphere_processes']['p3']) - p3 = pyp3.P3(grid,p3params) + p3params = pyeamxx.ParameterList(yaml_input['atmosphere_processes']['p3'],'p3') + p3 = pyeamxx.AtmProc(p3params) missing = p3.read_ic(str(ic_file)) - print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") + if len(missing)>0: + print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") p3.initialize(t0_str) p3.setup_output("output_py.yaml") @@ -57,7 +57,7 @@ if __name__ == "__main__": # This level of indirection ensures all pybind structs are destroyed # before we finalize eamxx (and hence kokkos) MPI.Init() - ps.init() + pyeamxx.init() main () - ps.finalize() + pyeamxx.finalize() MPI.Finalize() From 1b69dd67b323c05cea912ca3daf5a10cd83f11bd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 14:03:27 -0600 Subject: [PATCH 039/477] PyEAMxx: register all procs/diags during pyeamxx init --- components/eamxx/src/python/pyatmproc.hpp | 6 +----- components/eamxx/src/python/pyeamxx.cpp | 5 +++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index f4daf87233ff..cbf4a1706f8d 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -1,8 +1,6 @@ #ifndef PYATMPROC_HPP #define PYATMPROC_HPP -#include "physics/register_physics.hpp" -#include "diagnostics/register_diagnostics.hpp" #include "share/atm_process/atmosphere_process.hpp" #include "share/io/scorpio_input.hpp" #include "share/io/scream_output_manager.hpp" @@ -34,7 +32,6 @@ struct PyAtmProc { const auto& comm = PySession::get().comm; // Create the atm proc - register_physics(); auto& apf = AtmosphereProcessFactory::instance(); const auto& ap_type = params.pl.isParameter("Type") ? params.pl.get("Type") @@ -146,8 +143,7 @@ struct PyAtmProc { fms.at(gn)->add_field(it.second.f); } - // Make sure diagnostics are available, then create the output mgr - register_diagnostics(); + // Create/setup the output mgr output_mgr = std::make_shared(); output_mgr->setup(comm,params,fms,gm,t0,t0,false); output_mgr->set_logger(ap->get_logger()); diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index 93a1e0ab2dfa..c4ec360d80a1 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -20,6 +20,11 @@ void initialize (MPI_Comm mpi_comm) { initialize_scream_session(comm.am_i_root()); scorpio::init_subsystem(comm); + // Register everything in the eamxx factories + register_physics(); + register_dynamics(); + register_diagnostics(); + auto& s = PySession::get(); s.comm = comm; s.inited = true; From b0f4ab07863c3bcab6a753e5c5eba349aa3159df Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 14:13:07 -0600 Subject: [PATCH 040/477] PyEAMxx: prune unused tests directory --- .../eamxx/src/python/tests/basic_test.py | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 components/eamxx/src/python/tests/basic_test.py diff --git a/components/eamxx/src/python/tests/basic_test.py b/components/eamxx/src/python/tests/basic_test.py deleted file mode 100644 index 26e43cad0d2b..000000000000 --- a/components/eamxx/src/python/tests/basic_test.py +++ /dev/null @@ -1,20 +0,0 @@ -# luca's verbatim example works from any dir now! -from pyeamxx import pyscream as ps -import numpy as np - -ps.init() - -v = np.zeros(shape=(10,2)) - -for i in range(0,10): - for j in range(0,2): - v[i,j] = i*2 + j + 1 - -print(f"v:{v}") - -f = ps.Field("T_mid",v) - -f.print() - -f.cleanup() -ps.finalize() From 19ad5048f55f3a77d62757e6f54446937162466d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 16:35:20 -0600 Subject: [PATCH 041/477] PyEAMxx: add view semantic to PyParamList Allows to store reference_wrapper to existing parameter list --- components/eamxx/src/python/pyparamlist.hpp | 53 +++++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/pyparamlist.hpp index 00ee1c894f84..5d31a2c64274 100644 --- a/components/eamxx/src/python/pyparamlist.hpp +++ b/components/eamxx/src/python/pyparamlist.hpp @@ -7,19 +7,54 @@ #include #include +#include + namespace scream { struct PyParamList { ekat::ParameterList pl; + std::reference_wrapper pl_ref; + + PyParamList (ekat::ParameterList& src) + : pl_ref(src) + {} - PyParamList(const pybind11::dict& d) { + PyParamList(const pybind11::dict& d) + : PyParamList(d,"") + {} + + PyParamList(const pybind11::dict& d, const std::string& name) + : pl(name) + , pl_ref(pl) + { parse_dict(d,pl); } - PyParamList(const pybind11::dict& d, const std::string& name) { - pl.rename(name); - parse_dict(d,pl); + PyParamList sublist (const std::string& name) { + PyParamList spl(pl.sublist(name)); + return spl; + } + + int get_int (const std::string& name) const { + return pl_ref.get().get(name); + } + double get_dbl (const std::string& name) const { + return pl_ref.get().get(name); } + std::string get_str (const std::string& name) const { + return pl_ref.get().get(name); + } + + template + void set (const std::string& name, T val) { + pl_ref.get().set(name,val); + } + + void print () { + pl_ref.get().print(); + } + +private: void parse_dict(const pybind11::dict& d, ekat::ParameterList& p) { for (auto item : d) { @@ -84,7 +119,15 @@ inline void pybind_pyparamlist (pybind11::module& m) // Param list pybind11::class_(m,"ParameterList") .def(pybind11::init()) - .def(pybind11::init()); + .def(pybind11::init()) + .def("sublist",&PyParamList::sublist) + .def("print",&PyParamList::print) + .def("set",&PyParamList::set) + .def("set",&PyParamList::set) + .def("set",&PyParamList::set) + .def("get_int",&PyParamList::get_int) + .def("get_dbl",&PyParamList::get_dbl) + .def("get_str",&PyParamList::get_str); } } // namespace scream From 9c327e8c72d5e88f4fa64608954560c80613e064 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 16:36:09 -0600 Subject: [PATCH 042/477] PyEAMxx: changed AtmProc constructor to build straight from a dict --- components/eamxx/src/python/pyatmproc.hpp | 6 ++++-- components/eamxx/tests/python/pyp3/p3_standalone_py | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index cbf4a1706f8d..cc697c0e6c0b 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -26,8 +26,10 @@ struct PyAtmProc { std::shared_ptr output_mgr; - PyAtmProc (const PyParamList& params) + PyAtmProc (const pybind11::dict& d, const std::string& name) { + PyParamList params(d,name); + // Get the comm const auto& comm = PySession::get().comm; @@ -162,7 +164,7 @@ struct PyAtmProc { inline void pybind_pyatmproc(pybind11::module& m) { pybind11::class_(m,"AtmProc") - .def(pybind11::init()) + .def(pybind11::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) .def("setup_output",&PyAtmProc::setup_output) diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index def3c7f01b6d..c6a02d27010a 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -40,8 +40,7 @@ def main (): nlevs = 72 pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) - p3params = pyeamxx.ParameterList(yaml_input['atmosphere_processes']['p3'],'p3') - p3 = pyeamxx.AtmProc(p3params) + p3 = pyeamxx.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') missing = p3.read_ic(str(ic_file)) if len(missing)>0: print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") From cab6fa3d69acead6b50f824e9e7aa87bf54bf9dc Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 25 Jun 2024 16:37:52 -0600 Subject: [PATCH 043/477] PyEAMxx: allow to set/get parameters into/from a py AtmProc --- components/eamxx/src/python/pyatmproc.hpp | 6 ++++++ components/eamxx/tests/python/pyp3/p3_standalone_py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index cc697c0e6c0b..98d7b9a525ed 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -49,6 +49,11 @@ struct PyAtmProc { // I don't think virtual is needed, but just in case virtual ~PyAtmProc () = default; + PyParamList get_params () const { + PyParamList pypl(ap->get_params()); + return pypl; + } + void create_fields () { // Create fields that are input/output to the atm proc for (const auto& req : ap->get_required_field_requests()) { @@ -167,6 +172,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def(pybind11::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) + .def("get_params",&PyAtmProc::get_params) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) .def("read_ic",&PyAtmProc::read_ic); diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index c6a02d27010a..0db6bcfcef61 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -41,6 +41,14 @@ def main (): pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) p3 = pyeamxx.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') + params = p3.get_params() + old = params.get_dbl('max_total_ni') + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") + params.set("max_total_ni",1000000.0) + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") + params.set("max_total_ni",old) + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") + missing = p3.read_ic(str(ic_file)) if len(missing)>0: print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") From 0de9b423a8066e0e4c7a03c6ee84f3be7d6abfc5 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 11:57:53 -0600 Subject: [PATCH 044/477] PyEAMxx: add support for vector parameters in PyParamList --- components/eamxx/src/python/pyparamlist.hpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/pyparamlist.hpp index 5d31a2c64274..3afc0a1cf1af 100644 --- a/components/eamxx/src/python/pyparamlist.hpp +++ b/components/eamxx/src/python/pyparamlist.hpp @@ -45,6 +45,16 @@ struct PyParamList { return pl_ref.get().get(name); } + std::vector get_int_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + std::vector get_dbl_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + std::vector get_str_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + template void set (const std::string& name, T val) { pl_ref.get().set(name,val); @@ -125,9 +135,15 @@ inline void pybind_pyparamlist (pybind11::module& m) .def("set",&PyParamList::set) .def("set",&PyParamList::set) .def("set",&PyParamList::set) + .def("set",&PyParamList::set>) + .def("set",&PyParamList::set>) + .def("set",&PyParamList::set>) .def("get_int",&PyParamList::get_int) .def("get_dbl",&PyParamList::get_dbl) - .def("get_str",&PyParamList::get_str); + .def("get_str",&PyParamList::get_str) + .def("get_int_vec",&PyParamList::get_int_vec) + .def("get_dbl_vec",&PyParamList::get_dbl_vec) + .def("get_str_vec",&PyParamList::get_str_vec); } } // namespace scream From 0cbf490490aa4357e719553c622f015d5468f10b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 11:59:21 -0600 Subject: [PATCH 045/477] PyEAMxx: add support for bool params in PyParamList --- components/eamxx/src/python/pyparamlist.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/pyparamlist.hpp index 3afc0a1cf1af..a631b6622e90 100644 --- a/components/eamxx/src/python/pyparamlist.hpp +++ b/components/eamxx/src/python/pyparamlist.hpp @@ -35,6 +35,9 @@ struct PyParamList { return spl; } + bool get_bool (const std::string& name) const { + return pl_ref.get().get(name); + } int get_int (const std::string& name) const { return pl_ref.get().get(name); } @@ -132,12 +135,14 @@ inline void pybind_pyparamlist (pybind11::module& m) .def(pybind11::init()) .def("sublist",&PyParamList::sublist) .def("print",&PyParamList::print) + .def("set",&PyParamList::set) .def("set",&PyParamList::set) .def("set",&PyParamList::set) .def("set",&PyParamList::set) .def("set",&PyParamList::set>) .def("set",&PyParamList::set>) .def("set",&PyParamList::set>) + .def("get_bool",&PyParamList::get_bool) .def("get_int",&PyParamList::get_int) .def("get_dbl",&PyParamList::get_dbl) .def("get_str",&PyParamList::get_str) From 9498dcc9a6ccb1784a3ea67d9b55e71061312010 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 14:10:41 -0600 Subject: [PATCH 046/477] EAMxx: remove pointless stuff from scorpio_input.hpp --- .../share/grid/mesh_free_grids_manager.cpp | 1 + .../share/grid/remap/vertical_remapper.cpp | 3 ++- .../eamxx/src/share/io/scorpio_input.hpp | 19 ++----------------- .../iop/intensive_observation_period.cpp | 1 + .../share/util/eamxx_time_interpolation.cpp | 1 + 5 files changed, 7 insertions(+), 18 deletions(-) diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index e7e11ef20e2a..c930013f8398 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -4,6 +4,7 @@ #include "share/grid/remap/do_nothing_remapper.hpp" #include "share/property_checks/field_nan_check.hpp" #include "share/property_checks/field_within_interval_check.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/io/scorpio_input.hpp" #include "physics/share/physics_constants.hpp" diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 5e714c48aa0d..9b524cec5e49 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -5,8 +5,9 @@ #include "share/field/field_tag.hpp" #include "share/field/field_identifier.hpp" #include "share/util/scream_universal_constants.hpp" +#include "share/io/scream_scorpio_interface.hpp" -#include "ekat/util/ekat_units.hpp" +#include #include #include #include diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 801d224be831..2c0a7e76c3eb 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -1,10 +1,8 @@ #ifndef SCREAM_SCORPIO_INPUT_HPP #define SCREAM_SCORPIO_INPUT_HPP -#include "share/io/scream_scorpio_interface.hpp" #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" -#include "share/grid/grids_manager.hpp" #include "ekat/ekat_parameter_list.hpp" #include "ekat/logging/ekat_logger.hpp" @@ -23,22 +21,11 @@ * ----- * Input Parameters * Filename: STRING - * Fields: ARRAY OF STRINGS + * Field Names: ARRAY OF STRINGS * ----- * The meaning of these parameters is the following: * - Filename: the name of the input file to be read. - * - Fields: list of names of fields to load from file. Should match the name in the file and the name in the field manager. - * Note: you can specify lists (such as the 'Fields' list above) with either of the two syntaxes - * Fields: [field_name1, field_name2, ... , field_name_N] - * Fields: - * - field_name_1 - * - field_name_2 - * ... - * - field_name_N - * Note: an alternative way of specifying Fields names is to have - * Grid: STRING - * Fields: - * $GRID: [field_name1,...,field_name_N] + * - Field Names: list of names of fields to load from file. Should match the name in the file and the name in the field manager. * * TODO: add a rename option if variable names differ in file and field manager. * @@ -55,8 +42,6 @@ class AtmosphereInput public: using fm_type = FieldManager; using grid_type = AbstractGrid; - using gm_type = GridsManager; - using remapper_type = AbstractRemapper; using KT = KokkosTypes; template diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index bbe3306a415e..657f7ad742a0 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -1,5 +1,6 @@ #include "share/grid/point_grid.hpp" #include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/iop/intensive_observation_period.hpp" #include "ekat/ekat_assert.hpp" diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index e0dba0cf8382..aa46328acac3 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -1,4 +1,5 @@ #include "share/util/eamxx_time_interpolation.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/io/scream_io_utils.hpp" namespace scream{ From 07955c6851987ee59620598316e819b4bd466462 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 14:11:38 -0600 Subject: [PATCH 047/477] EAMxx: add input file reader for a single-column model --- components/eamxx/src/share/io/CMakeLists.txt | 1 + .../eamxx/src/share/io/scorpio_scm_input.cpp | 246 ++++++++++++++++++ .../eamxx/src/share/io/scorpio_scm_input.hpp | 68 +++++ 3 files changed, 315 insertions(+) create mode 100644 components/eamxx/src/share/io/scorpio_scm_input.cpp create mode 100644 components/eamxx/src/share/io/scorpio_scm_input.hpp diff --git a/components/eamxx/src/share/io/CMakeLists.txt b/components/eamxx/src/share/io/CMakeLists.txt index afb0ef888db7..6c24d3bc51da 100644 --- a/components/eamxx/src/share/io/CMakeLists.txt +++ b/components/eamxx/src/share/io/CMakeLists.txt @@ -54,6 +54,7 @@ endif() add_library(scream_io scream_output_manager.cpp scorpio_input.cpp + scorpio_scm_input.cpp scorpio_output.cpp scream_io_utils.cpp ) diff --git a/components/eamxx/src/share/io/scorpio_scm_input.cpp b/components/eamxx/src/share/io/scorpio_scm_input.cpp new file mode 100644 index 000000000000..c5a4b9149ae9 --- /dev/null +++ b/components/eamxx/src/share/io/scorpio_scm_input.cpp @@ -0,0 +1,246 @@ +#include "share/io/scorpio_scm_input.hpp" +#include "share/io/scorpio_input.hpp" + +#include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/point_grid.hpp" + +#include + +#include + +namespace scream +{ + +SCMInput:: +SCMInput (const std::string& filename, + const double lat, const double lon, + const std::vector& fields, + const ekat::Comm& comm) + : m_comm(comm) + , m_filename (filename) +{ + auto iotype = scorpio::str2iotype("default"); + scorpio::register_file(m_filename,scorpio::Read,iotype); + + // Some input files have the "time" dimension as non-unlimited. This messes up our + // scorpio interface. To avoid trouble, if a dim called 'time' is present we + // treat it as unlimited, even though it isn't. + if (scorpio::has_dim(m_filename,"time") and not scorpio::is_dim_unlimited(m_filename,"time")) { + scorpio::pretend_dim_is_unlimited(m_filename,"time"); + } + + create_io_grid (); + auto ncols = m_io_grid->get_num_local_dofs(); + + create_closest_col_info (lat, lon); + + // Init fields specs + for (const auto& f : fields) { + const auto& fh = f.get_header(); + const auto& fid = fh.get_identifier(); + const auto& fl = fid.get_layout(); + + EKAT_REQUIRE_MSG (fl.tags()[0]==FieldTag::Column, + "Error! SCMInput only works for physics-type layouts.\n" + " - field name: " + f.name() + "\n" + " - field layout: " + fl.to_string() + "\n"); + + m_fields.push_back(f.subfield(0,0)); + FieldIdentifier fid_io(f.name(),fl.clone().reset_dim(0,ncols),fid.get_units(),m_io_grid->name()); + auto& f_io = m_io_fields.emplace_back(fid_io); + f_io.allocate_view(); + } + + // Init scorpio internal structures + init_scorpio_structures (); +} + +SCMInput:: +~SCMInput () +{ + scorpio::release_file(m_filename); +} + +void SCMInput::create_io_grid () +{ + EKAT_REQUIRE_MSG (scorpio::has_dim(m_filename,"ncol"), + "Error! Dimension 'ncol' not found in input file.\n" + " - filename: " + m_filename + "\n"); + const int ncols = scorpio::get_dimlen(m_filename,"ncol"); + const int nlevs = scorpio::has_dim(m_filename,"lev") ? scorpio::get_dimlen(m_filename,"lev") : 1; + + m_io_grid = create_point_grid("scm_io_grid",ncols,nlevs,m_comm); +} + +void SCMInput::create_closest_col_info (double target_lat, double target_lon) +{ + // Read lat/lon fields + const auto ncols = m_io_grid->get_num_local_dofs(); + + auto nondim = ekat::units::Units::nondimensional(); + auto lat = m_io_grid->create_geometry_data("lat",m_io_grid->get_2d_scalar_layout(),nondim); + auto lon = m_io_grid->create_geometry_data("lon",m_io_grid->get_2d_scalar_layout(),nondim); + + // Read from file + AtmosphereInput file_reader(m_filename, m_io_grid, {lat,lon}); + file_reader.read_variables(); + file_reader.finalize(); + + // Find column index of closest lat/lon to target_lat/lon params + auto lat_d = lat.get_view(); + auto lon_d = lon.get_view(); + using minloc_t = Kokkos::MinLoc; + using minloc_value_t = typename minloc_t::value_type; + minloc_value_t minloc; + Kokkos::parallel_reduce(ncols, KOKKOS_LAMBDA (int icol, minloc_value_t& result) { + auto dist = std::abs(lat_d(icol)-target_lat)+std::abs(lon_d(icol)-target_lon); + if(dist min_dist_and_rank = {minloc.val, my_rank}; + m_comm.all_reduce>(&min_dist_and_rank, 1, MPI_MINLOC); + + // Set local col idx to -1 for mpi ranks not containing minimum lat/lon distance + m_closest_col_info.mpi_rank = min_dist_and_rank.second; + m_closest_col_info.col_lid = my_rank==min_dist_and_rank.second ? minloc.loc : -1; +} + +void SCMInput::read_variables (const int time_index) +{ + auto func_start = std::chrono::steady_clock::now(); + auto fname = [](const Field& f) { return f.name(); }; + if (m_atm_logger) { + m_atm_logger->info("[EAMxx::scorpio_scm_input] Reading variables from file"); + m_atm_logger->info(" file name: " + m_filename); + m_atm_logger->info(" var names: " + ekat::join(m_fields,fname,", ")); + if (time_index!=-1) { + m_atm_logger->info(" time idx : " + std::to_string(time_index)); + } + } + + // MPI rank with closest column index store column data + const int n = m_fields.size(); + for (int i=0; i(),time_index); + + auto& f = m_fields[i]; + if (m_comm.rank() == m_closest_col_info.mpi_rank) { + // This rank read the column we need, so copy it in the output field + f.deep_copy(f_io.subfield(0,m_closest_col_info.col_lid)); + } + + // Broadcast column data to all other ranks + const auto col_size = f.get_header().get_identifier().get_layout().size(); + m_comm.broadcast(f.get_internal_view_data(), col_size, m_closest_col_info.mpi_rank); + + // Sync fields to device + f.sync_to_dev(); + } + + auto func_finish = std::chrono::steady_clock::now(); + if (m_atm_logger) { + auto duration = std::chrono::duration_cast(func_finish - func_start)/1000.0; + m_atm_logger->info(" Done! Elapsed time: " + std::to_string(duration.count()) +" seconds"); + } +} + +void SCMInput::init_scorpio_structures() +{ + using namespace ShortFieldTagsNames; + + // Check variables are in the input file + for (const auto& f : m_io_fields) { + const auto& layout = f.get_header().get_identifier().get_layout(); + auto dim_names = layout.names(); + for (int i=0; iget_partitioned_dim_global_size() : layout.dim(i); + EKAT_REQUIRE_MSG (eamxx_len==file_len, + "Error! Dimension mismatch for input file variable.\n" + " - filename : " + m_filename + "\n" + " - varname : " + f.name() + "\n" + " - var dims : " + ekat::join(dim_names,",") + "\n" + " - dim name : " + dim_names[i] + "\n" + " - expected extent : " + std::to_string(eamxx_len) + "\n" + " - extent from file: " + std::to_string(file_len) + "\n"); + } + + // Ensure that we can read the var using Real data type + scorpio::change_var_dtype (m_filename,f.name(),"real"); + } + + // Set decompositions for the variables + set_decompositions(); +} + +/* ---------------------------------------------------------- */ +void SCMInput::set_decompositions() +{ + using namespace ShortFieldTagsNames; + + // First, check if any of the vars is indeed partitioned + const auto decomp_tag = m_io_grid->get_partitioned_dim_tag(); + + bool has_decomposed_layouts = false; + for (const auto& f : m_io_fields) { + const auto& layout = f.get_header().get_identifier().get_layout(); + if (layout.has_tag(decomp_tag)) { + has_decomposed_layouts = true; + break; + } + } + if (not has_decomposed_layouts) { + // If none of the input vars are decomposed on this grid, + // then there's nothing to do here + return; + } + + // Set the decomposition for the partitioned dimension + const int local_dim = m_io_grid->get_partitioned_dim_local_size(); + std::string decomp_dim = m_io_grid->has_special_tag_name(decomp_tag) + ? m_io_grid->get_special_tag_name(decomp_tag) + : e2str(decomp_tag); + + auto gids_f = m_io_grid->get_partitioned_dim_gids(); + auto gids_h = gids_f.get_view(); + auto min_gid = m_io_grid->get_global_min_partitioned_dim_gid(); + std::vector offsets(local_dim); + for (int idof=0; idof + +namespace scream +{ + +// Similar to AtmosphereInput, but reads in a single column from +// a file with N columns. A few assumptions: +// - lat and lon variables are present in the file +// - fields have layout +class SCMInput +{ +public: + // --- Constructor(s) & Destructor --- // + SCMInput (const std::string& filename, + const double lat, const double lon, + const std::vector& fields, + const ekat::Comm& comm); + + ~SCMInput (); + + // Due to resource acquisition (in scorpio), avoid copies + SCMInput (const SCMInput&) = delete; + SCMInput& operator= (const SCMInput&) = delete; + + // Read fields that were required via parameter list. + void read_variables (const int time_index = -1); + + // Option to add a logger + void set_logger(const std::shared_ptr& atm_logger) { + m_atm_logger = atm_logger; + } +protected: + + struct ClosestColInfo { + // MPI rank which owns the columns whose lat/lon pair is the closest to target lat/lon + int mpi_rank; + // Local column index of on rank=mpi_rank (-1 on all other ranks) + int col_lid; + }; + + void create_io_grid (); + void create_closest_col_info (double target_lat, double target_lon); + void init_scorpio_structures (); + void set_decompositions(); + + // Internal variables + ekat::Comm m_comm; + + std::shared_ptr m_io_grid; + + std::string m_filename; + std::vector m_fields; + std::vector m_io_fields; + + ClosestColInfo m_closest_col_info; + + // The logger to be used throughout the ATM to log message + std::shared_ptr m_atm_logger; +}; + +} //namespace scream + +#endif // SCREAM_SCORPIO_SCM_INPUT_HPP From 49ad7b36a397de7b63d831f6751abc4bec32e8cc Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 26 Jun 2024 15:41:55 -0600 Subject: [PATCH 048/477] EAMxx: add unit test for SCMInput --- .../eamxx/src/share/io/tests/CMakeLists.txt | 6 + .../src/share/io/tests/io_scm_reader.cpp | 130 ++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 components/eamxx/src/share/io/tests/io_scm_reader.cpp diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index 16ce2efd4376..8c819f7896cc 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -81,3 +81,9 @@ CreateUnitTest(io_remap_test "io_remap_test.cpp" LIBS scream_io diagnostics LABELS io remap MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) + +## Test single-column reader +CreateUnitTest(io_scm_reader "io_scm_reader.cpp" + LIBS scream_io LABELS io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} +) diff --git a/components/eamxx/src/share/io/tests/io_scm_reader.cpp b/components/eamxx/src/share/io/tests/io_scm_reader.cpp new file mode 100644 index 000000000000..403d557582a9 --- /dev/null +++ b/components/eamxx/src/share/io/tests/io_scm_reader.cpp @@ -0,0 +1,130 @@ +#include + +#include "share/io/scorpio_scm_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/field/field.hpp" +#include "share/field/field_utils.hpp" + +#include "share/util/scream_setup_random_test.hpp" + +namespace scream { + +// Returns fields after initialization +void write (const int seed, const int ncols, const int nlevs, const ekat::Comm& comm) +{ + using ekat::units::Units; + using RPDF = std::uniform_real_distribution; + using Engine = std::mt19937_64; + + Engine engine(seed); + RPDF lat_pdf(-90.0,90.0); + RPDF lon_pdf(-180.0,180.0); + + // Create grid + auto grid = create_point_grid("test",ncols,nlevs,comm); + + // Create lat/lon grid + Units deg (Units::nondimensional(),"deg"); + auto lat = grid->create_geometry_data("lat",grid->get_2d_scalar_layout(),deg); + auto lon = grid->create_geometry_data("lon",grid->get_2d_scalar_layout(),deg); + randomize(lat,engine,lat_pdf); + randomize(lon,engine,lon_pdf); + + // Create variable data + FieldIdentifier fid("var",grid->get_3d_scalar_layout(true),Units::nondimensional(),""); + Field var(fid); + var.allocate_view(); + randomize(var,engine,RPDF(-1.0,1.0)); + + // Create file + auto filename = "io_scm_np" + std::to_string(comm.size()) + ".nc"; + scorpio::register_file(filename,scorpio::Write,scorpio::DefaultIOType); + + scorpio::define_dim(filename,"ncol",ncols); + scorpio::define_dim(filename,"lev",nlevs); + + scorpio::define_var(filename,"lat",{"ncol"},"real"); + scorpio::define_var(filename,"lon",{"ncol"},"real"); + + scorpio::define_var(filename,"var",{"ncol","lev"},"real"); + + auto my_col_gids = grid->get_partitioned_dim_gids().get_view(); + std::vector my_offsets(my_col_gids.size()); + for (size_t i=0; iget_global_min_partitioned_dim_gid (); + } + scorpio::set_dim_decomp(filename,"ncol",my_offsets); + scorpio::enddef(filename); + + // Write to file + scorpio::write_var(filename,"lat",lat.get_internal_view_data()); + scorpio::write_var(filename,"lon",lon.get_internal_view_data()); + scorpio::write_var(filename,"var",var.get_internal_view_data()); + + scorpio::release_file(filename); +} + +void read (const int seed, const int nlevs, const ekat::Comm& comm) +{ + using ekat::units::Units; + using IPDF = std::uniform_int_distribution; + using Engine = std::mt19937_64; + + Engine engine(seed); + + auto grid = create_point_grid("scm_grid",1,nlevs,comm); + + // Pick a col index, and find its lat/lon from file + auto filename = "io_scm_np" + std::to_string(comm.size()) + ".nc"; + scorpio::register_file(filename,scorpio::Read,scorpio::DefaultIOType); + int ncols = scorpio::get_dimlen(filename,"ncol"); + + std::vector lat(ncols), lon(ncols), var(ncols*nlevs); + scorpio::read_var(filename,"lat",lat.data()); + scorpio::read_var(filename,"lon",lon.data()); + scorpio::read_var(filename,"var",var.data()); + + scorpio::release_file(filename); + + auto tgt_col = IPDF(0,ncols-1)(engine); + + auto tgt_lat = lat[tgt_col]; + auto tgt_lon = lon[tgt_col]; + + // Create field to read + FieldIdentifier fid("var",grid->get_3d_scalar_layout(true),Units::nondimensional(),""); + Field var_f(fid); + var_f.allocate_view(); + + // Read field + SCMInput reader(filename,tgt_lat,tgt_lon,{var_f},comm); + reader.read_variables(); + + // Check + auto var_h = var_f.get_view(); + for (int ilev=0; ilev; + using Engine = std::mt19937_64; + + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::init_subsystem(comm); + + auto seed = get_random_test_seed(&comm); + Engine engine(seed); + const int nlevs = IPDF(5,50)(engine); + const int ncols = IPDF(20,50)(engine); + + write(seed,ncols,nlevs,comm); + read(seed,nlevs,comm); + + scorpio::finalize_subsystem(); +} + +} // anonymous namespace From d39248d423fb02ab7e678b3aa464f8dbc8879b51 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 27 Jun 2024 09:52:56 -0600 Subject: [PATCH 049/477] EAMxx: add missing include in nudging interface --- .../src/physics/nudging/eamxx_nudging_process_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 3af33eaea164..8ebfc9c14c01 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -4,6 +4,7 @@ #include "share/grid/remap/refining_remapper_p2p.hpp" #include "share/grid/remap/do_nothing_remapper.hpp" #include "share/util/scream_utils.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include #include From 5ffbba0f7817813a37f797a95348d8c7e223e74b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 27 Jun 2024 14:35:15 -0600 Subject: [PATCH 050/477] EAMxx: fix scm reader unit test --- .../eamxx/src/share/io/tests/io_scm_reader.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/io/tests/io_scm_reader.cpp b/components/eamxx/src/share/io/tests/io_scm_reader.cpp index 403d557582a9..111f82947504 100644 --- a/components/eamxx/src/share/io/tests/io_scm_reader.cpp +++ b/components/eamxx/src/share/io/tests/io_scm_reader.cpp @@ -74,7 +74,9 @@ void read (const int seed, const int nlevs, const ekat::Comm& comm) Engine engine(seed); - auto grid = create_point_grid("scm_grid",1,nlevs,comm); + auto grid = std::make_shared("scm_grid",1,nlevs,comm); + auto dofs_gids = grid->get_dofs_gids(); + dofs_gids.deep_copy(0); // Pick a col index, and find its lat/lon from file auto filename = "io_scm_np" + std::to_string(comm.size()) + ".nc"; @@ -89,6 +91,7 @@ void read (const int seed, const int nlevs, const ekat::Comm& comm) scorpio::release_file(filename); auto tgt_col = IPDF(0,ncols-1)(engine); + comm.broadcast(&tgt_col,1,comm.root_rank()); auto tgt_lat = lat[tgt_col]; auto tgt_lon = lon[tgt_col]; @@ -118,8 +121,11 @@ TEST_CASE ("scm_io") { auto seed = get_random_test_seed(&comm); Engine engine(seed); - const int nlevs = IPDF(5,50)(engine); - const int ncols = IPDF(20,50)(engine); + int nlevs = IPDF(5,8)(engine); + int ncols = IPDF(4,5)(engine); + + comm.broadcast(&ncols,1,comm.root_rank()); + comm.broadcast(&nlevs,1,comm.root_rank()); write(seed,ncols,nlevs,comm); read(seed,nlevs,comm); From 241899b1d07b0677f85f7b19254a623f3b817003 Mon Sep 17 00:00:00 2001 From: noel Date: Thu, 27 Jun 2024 17:46:54 -0700 Subject: [PATCH 051/477] revert back to hex hash prints --- .../src/share/atm_process/atmosphere_process_hash.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 6b2f2fd080bb..37cb251d7796 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,9 +129,9 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lld (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), - i, (long long int)gaccum[i], label.c_str()); + i, gaccum[i], label.c_str()); } void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) const { @@ -140,8 +140,8 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lld (%s)\n", - timestamp().get_num_steps(), (long long int) gaccum, label.c_str()); + fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + timestamp().get_num_steps(), gaccum, label.c_str()); } } // namespace scream From c195b3c01f60922f5ff36fab65ec02560807a37f Mon Sep 17 00:00:00 2001 From: noel Date: Fri, 28 Jun 2024 11:51:26 -0700 Subject: [PATCH 052/477] use llx instead of lx for print format --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 37cb251d7796..9fb5726084f4 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } From f9a89377f327ba15cf20c637f237bd26f9cafe3a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 1 Jul 2024 09:52:35 -0600 Subject: [PATCH 053/477] EAMxx: fix CUDA issue in scm reader --- components/eamxx/src/share/io/scorpio_scm_input.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scorpio_scm_input.hpp b/components/eamxx/src/share/io/scorpio_scm_input.hpp index 55b4c637cb1b..44b54043dcad 100644 --- a/components/eamxx/src/share/io/scorpio_scm_input.hpp +++ b/components/eamxx/src/share/io/scorpio_scm_input.hpp @@ -34,6 +34,12 @@ class SCMInput void set_logger(const std::shared_ptr& atm_logger) { m_atm_logger = atm_logger; } + +#ifndef KOKKOS_ENABLE_CUDA + // Cuda requires methods enclosing __device__ lambda's to be public +protected: +#endif + void create_closest_col_info (double target_lat, double target_lon); protected: struct ClosestColInfo { @@ -44,7 +50,6 @@ class SCMInput }; void create_io_grid (); - void create_closest_col_info (double target_lat, double target_lon); void init_scorpio_structures (); void set_decompositions(); From e43b59f1b378f096d83682aebacc3e10dad361f0 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 3 May 2024 18:39:16 -0400 Subject: [PATCH 054/477] add hacky atm tend diag --- .../eamxx/src/diagnostics/CMakeLists.txt | 1 + components/eamxx/src/diagnostics/atm_tend.cpp | 76 ++++++++++++ components/eamxx/src/diagnostics/atm_tend.hpp | 50 ++++++++ .../src/diagnostics/register_diagnostics.hpp | 2 + .../src/diagnostics/tests/CMakeLists.txt | 3 + .../src/diagnostics/tests/atm_tend_test.cpp | 111 ++++++++++++++++++ .../eamxx/src/share/io/scorpio_output.cpp | 7 ++ 7 files changed, 250 insertions(+) create mode 100644 components/eamxx/src/diagnostics/atm_tend.cpp create mode 100644 components/eamxx/src/diagnostics/atm_tend.hpp create mode 100644 components/eamxx/src/diagnostics/tests/atm_tend_test.cpp diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index a7a1ee9eaeaf..75c6f420fc61 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -20,6 +20,7 @@ set(DIAGNOSTIC_SRCS aodvis.cpp number_path.cpp aerocom_cld.cpp + atm_tend.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/atm_tend.cpp b/components/eamxx/src/diagnostics/atm_tend.cpp new file mode 100644 index 000000000000..e4903e6d5573 --- /dev/null +++ b/components/eamxx/src/diagnostics/atm_tend.cpp @@ -0,0 +1,76 @@ +#include "diagnostics/atm_tend.hpp" + +#include + +#include "share/util/scream_universal_constants.hpp" + +namespace scream { + +AtmTendDiag::AtmTendDiag(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereDiagnostic(comm, params) { + EKAT_REQUIRE_MSG(params.isParameter("Tend Name"), + "Error! AtmTendDiag requires 'Tend Name' in its " + "input parameters.\n"); + + m_name = m_params.get("Tend Name"); +} + +std::string AtmTendDiag::name() const { return m_name + "_atm_tend"; } + +void AtmTendDiag::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + const auto &gname = m_params.get("grid_name"); + add_field(m_name, gname); +} + +void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { + const auto &f = get_field_in(m_name); + const auto &fid = f.get_header().get_identifier(); + + // Sanity checks + using namespace ShortFieldTagsNames; + const auto &layout = fid.get_layout(); + EKAT_REQUIRE_MSG(f.data_type() == DataType::RealType, + "Error! FieldAtHeight only supports Real data type field.\n" + " - field name: " + + fid.name() + + "\n" + " - field data type: " + + e2str(f.data_type()) + "\n"); + + // All good, create the diag output + FieldIdentifier d_fid(name(), layout.clone(), fid.get_units(), + fid.get_grid_name()); + m_diagnostic_output = Field(d_fid); + m_diagnostic_output.allocate_view(); + + // Let's also create the previous field + FieldIdentifier prev_fid(name() + "_prev", layout.clone(), fid.get_units(), + fid.get_grid_name()); + m_field_prev = Field(prev_fid); + m_field_prev.allocate_view(); +} +void AtmTendDiag::compute_diagnostic_impl() { + Real var_fill_value = constants::DefaultFillValue().value; + std::int64_t dt; + auto tts = m_diagnostic_output.get_header().get_tracking().get_time_stamp(); + + const auto &f = get_field_in(m_name); + + if(m_ts.is_valid()) { + dt = tts - m_ts; + auto ddt = static_cast(dt); + m_ts = tts; + m_field_prev.update(f, 1 / ddt, -1 / ddt); + m_diagnostic_output.deep_copy(m_field_prev); + } else { + m_diagnostic_output.deep_copy(var_fill_value); + m_ts = tts; + } + m_field_prev.deep_copy(f); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/atm_tend.hpp b/components/eamxx/src/diagnostics/atm_tend.hpp new file mode 100644 index 000000000000..f134fd3f6857 --- /dev/null +++ b/components/eamxx/src/diagnostics/atm_tend.hpp @@ -0,0 +1,50 @@ +#ifndef EAMXX_ATM_TEND_DIAG_HPP +#define EAMXX_ATM_TEND_DIAG_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/util/scream_time_stamp.hpp" + +namespace scream { + +/* + * This diagnostic will produce the atmospheric tendency. + */ + +class AtmTendDiag : public AtmosphereDiagnostic { + public: + // Constructors + AtmTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const; + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + protected: + void initialize_impl(const RunType /*run_type*/); + + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + // The tendency of what? + std::string m_name; + + // Store the previous field + Field m_field_prev; + + // Store a time stamp + util::TimeStamp m_ts; + +}; // class AtmTendDiag + +} // namespace scream + +#endif // EAMXX_ATM_TEND_DIAG_HPP diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index efb55980a2fb..07f90624db03 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -23,6 +23,7 @@ #include "diagnostics/aodvis.hpp" #include "diagnostics/number_path.hpp" #include "diagnostics/aerocom_cld.hpp" +#include "diagnostics/atm_tend.hpp" namespace scream { @@ -53,6 +54,7 @@ inline void register_diagnostics () { diag_factory.register_product("AerosolOpticalDepth550nm",&create_atmosphere_diagnostic); diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); diag_factory.register_product("AeroComCld",&create_atmosphere_diagnostic); + diag_factory.register_product("AtmTendDiag",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 2ea6bcffd88a..1dfb92fa7e29 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -71,4 +71,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) # Test AEROCOM_CLD CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") + # Test atm_tend + CreateDiagTest(atm_tend "atm_tend_test.cpp") + endif() diff --git a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp new file mode 100644 index 000000000000..5652215719ac --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp @@ -0,0 +1,111 @@ +#include "catch2/catch.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "share/field/field_utils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_universal_constants.hpp" + +namespace scream { + +std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, + const int nlevs) { + const int num_global_cols = ncols * comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names", vos_t{"Point Grid"}); + auto &pl = gm_params.sublist("Point Grid"); + pl.set("type", "point_grid"); + pl.set("aliases", vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm, gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("extraaci") { + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); + + const auto nondim = Units::nondimensional(); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 5; + const int ngcols = 1 * comm.size(); + + auto gm = create_gm(comm, ngcols, nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) qc, nc + FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; + FieldIdentifier qc_fid("qc", scalar2d_layout, kg / kg, grid->name()); + + Field qc(qc_fid); + qc.allocate_view(); + qc.get_header().get_tracking().update_time_stamp(t0); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(0, 0.05); + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto &diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + ekat::ParameterList params; + REQUIRE_THROWS( + diag_factory.create("AtmTendDiag", comm, params)); // No 'Tend Name' + + // TODO: The diag currently doesn't throw when given a phony name, need + // hardening! params.set("Tend Name", "NoWay"); REQUIRE_THROWS + // (diag_factory.create("AtmTendDiag",comm,params)); // Bad 'Tend Name' + + // Randomize + randomize(qc, engine, pdf); + // Create and set up the diagnostic + params.set("grid_name", grid->name()); + params.set("Tend Name", "qc"); + auto diag = diag_factory.create("AtmTendDiag", comm, params); + diag->set_grids(gm); + diag->set_required_field(qc); + diag->initialize(t0, RunType::Initial); + + auto qc_v = qc.get_view(); + qc_v(0, 0) = 5.0; + qc.sync_to_dev(); + + // Run diag + diag->compute_diagnostic(); + auto diag_f = diag->get_diagnostic(); + + Real var_fill_value = constants::DefaultFillValue().value; + + // Check result: diag should be filled with var_fill_value + auto some_field = qc.clone(); + some_field.deep_copy(var_fill_value); + REQUIRE(views_are_equal(diag_f, some_field)); + + util::TimeStamp t1({2024, 1, 2}, {0, 0, 0}); // a day later? + qc.get_header().get_tracking().update_time_stamp(t1); + qc_v(0, 0) = 29.0; + qc.sync_to_dev(); + // diag->initialize(t1, RunType::Initial); + // diag->update(t1, RunType::Initial); + diag->compute_diagnostic(); + diag_f = diag->get_diagnostic(); + auto diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 0) == 1.0 / 3600.0); +} + +} // namespace scream diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 3db201f424ba..8b7b861ff440 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1377,6 +1377,13 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { diag_name = "VaporFlux"; // split will return the list [X, ''], with X being whatever is before 'VapFlux' params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); + } else if (diag_field_name.find("_atm_tend")!=std::string::npos) { + diag_name = "AtmTendDiag"; + // TODO: not sure if this is needed? Can skip, but what to do inside diag? + // Set the grid_name + params.set("grid_name",get_field_manager("sim")->get_grid()->name()); + // split will return [X, ''], with X being whatever is before '_atm_tend' + params.set("Tend Name",ekat::split(diag_field_name,"_atm_tend").front()); } else if (diag_field_name=="PotentialTemperature" or diag_field_name=="LiqPotentialTemperature") { diag_name = "PotentialTemperature"; From 5025176dad03f4dddcff7c820237ebe80f2676d9 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 3 May 2024 23:32:44 -0400 Subject: [PATCH 055/477] address comments and harden tests --- components/eamxx/src/diagnostics/atm_tend.cpp | 47 +++++++++-------- components/eamxx/src/diagnostics/atm_tend.hpp | 5 +- .../src/diagnostics/tests/atm_tend_test.cpp | 51 +++++++++---------- 3 files changed, 52 insertions(+), 51 deletions(-) diff --git a/components/eamxx/src/diagnostics/atm_tend.cpp b/components/eamxx/src/diagnostics/atm_tend.cpp index e4903e6d5573..1420ea8f8478 100644 --- a/components/eamxx/src/diagnostics/atm_tend.cpp +++ b/components/eamxx/src/diagnostics/atm_tend.cpp @@ -9,11 +9,11 @@ namespace scream { AtmTendDiag::AtmTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereDiagnostic(comm, params) { - EKAT_REQUIRE_MSG(params.isParameter("Tend Name"), - "Error! AtmTendDiag requires 'Tend Name' in its " + EKAT_REQUIRE_MSG(params.isParameter("Field Name"), + "Error! AtmTendDiag requires 'Field Name' in its " "input parameters.\n"); - m_name = m_params.get("Tend Name"); + m_name = m_params.get("Field Name"); } std::string AtmTendDiag::name() const { return m_name + "_atm_tend"; } @@ -29,48 +29,55 @@ void AtmTendDiag::set_grids( void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { const auto &f = get_field_in(m_name); const auto &fid = f.get_header().get_identifier(); + const auto &gn = fid.get_grid_name(); // Sanity checks using namespace ShortFieldTagsNames; const auto &layout = fid.get_layout(); EKAT_REQUIRE_MSG(f.data_type() == DataType::RealType, - "Error! FieldAtHeight only supports Real data type field.\n" + "Error! AtmTendDiag only supports Real data type field.\n" " - field name: " + fid.name() + "\n" " - field data type: " + e2str(f.data_type()) + "\n"); + using namespace ekat::units; + // The units are the same except per second + auto diag_units = fid.get_units() / s; + // TODO: set the units string correctly by appending "/s" + // All good, create the diag output - FieldIdentifier d_fid(name(), layout.clone(), fid.get_units(), - fid.get_grid_name()); + FieldIdentifier d_fid(name(), layout.clone(), diag_units, gn); m_diagnostic_output = Field(d_fid); m_diagnostic_output.allocate_view(); // Let's also create the previous field - FieldIdentifier prev_fid(name() + "_prev", layout.clone(), fid.get_units(), - fid.get_grid_name()); - m_field_prev = Field(prev_fid); - m_field_prev.allocate_view(); + FieldIdentifier prev_fid(name() + "_prev", layout.clone(), diag_units, gn); + m_f_prev = Field(prev_fid); + m_f_prev.allocate_view(); } void AtmTendDiag::compute_diagnostic_impl() { Real var_fill_value = constants::DefaultFillValue().value; std::int64_t dt; - auto tts = m_diagnostic_output.get_header().get_tracking().get_time_stamp(); - const auto &f = get_field_in(m_name); + const auto &f = get_field_in(m_name); + const auto &curr_ts = f.get_header().get_tracking().get_time_stamp(); + const auto &prev_ts = m_f_prev.get_header().get_tracking().get_time_stamp(); - if(m_ts.is_valid()) { - dt = tts - m_ts; - auto ddt = static_cast(dt); - m_ts = tts; - m_field_prev.update(f, 1 / ddt, -1 / ddt); - m_diagnostic_output.deep_copy(m_field_prev); + if(prev_ts.is_valid()) { + // This diag was called before, so we have a valid value for m_f_prev, + // and can compute the tendency + dt = curr_ts - prev_ts; + m_f_prev.update(f, 1.0 / dt, -1.0 / dt); + m_diagnostic_output.deep_copy(m_f_prev); } else { + // This is the first time we evaluate this diag. We cannot compute a tend + // yet, so fill with an invalid value m_diagnostic_output.deep_copy(var_fill_value); - m_ts = tts; } - m_field_prev.deep_copy(f); + m_f_prev.deep_copy(f); + m_f_prev.get_header().get_tracking().update_time_stamp(curr_ts); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/atm_tend.hpp b/components/eamxx/src/diagnostics/atm_tend.hpp index f134fd3f6857..0cc707e7d4a5 100644 --- a/components/eamxx/src/diagnostics/atm_tend.hpp +++ b/components/eamxx/src/diagnostics/atm_tend.hpp @@ -38,10 +38,7 @@ class AtmTendDiag : public AtmosphereDiagnostic { std::string m_name; // Store the previous field - Field m_field_prev; - - // Store a time stamp - util::TimeStamp m_ts; + Field m_f_prev; }; // class AtmTendDiag diff --git a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp index 5652215719ac..79d1386bbaf8 100644 --- a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp +++ b/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp @@ -26,7 +26,7 @@ std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, return gm; } -TEST_CASE("extraaci") { +TEST_CASE("atm_tend") { using namespace ShortFieldTagsNames; using namespace ekat::units; @@ -36,11 +36,9 @@ TEST_CASE("extraaci") { // A time stamp util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); - const auto nondim = Units::nondimensional(); - // Create a grids manager - single column for these tests - constexpr int nlevs = 5; - const int ngcols = 1 * comm.size(); + constexpr int nlevs = 25; + const int ngcols = 25 * comm.size(); auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); @@ -51,11 +49,11 @@ TEST_CASE("extraaci") { Field qc(qc_fid); qc.allocate_view(); - qc.get_header().get_tracking().update_time_stamp(t0); // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(0, 0.05); + RPDF pdf(0.0, 200.0); + auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -65,47 +63,46 @@ TEST_CASE("extraaci") { ekat::ParameterList params; REQUIRE_THROWS( - diag_factory.create("AtmTendDiag", comm, params)); // No 'Tend Name' + diag_factory.create("AtmTendDiag", comm, params)); // No 'Field Name' - // TODO: The diag currently doesn't throw when given a phony name, need - // hardening! params.set("Tend Name", "NoWay"); REQUIRE_THROWS - // (diag_factory.create("AtmTendDiag",comm,params)); // Bad 'Tend Name' + Real var_fill_value = constants::DefaultFillValue().value; - // Randomize + // Set time for qc and randomize its values + qc.get_header().get_tracking().update_time_stamp(t0); randomize(qc, engine, pdf); + // Create and set up the diagnostic params.set("grid_name", grid->name()); - params.set("Tend Name", "qc"); + params.set("Field Name", "qc"); auto diag = diag_factory.create("AtmTendDiag", comm, params); diag->set_grids(gm); diag->set_required_field(qc); diag->initialize(t0, RunType::Initial); - auto qc_v = qc.get_view(); - qc_v(0, 0) = 5.0; - qc.sync_to_dev(); - // Run diag diag->compute_diagnostic(); auto diag_f = diag->get_diagnostic(); - Real var_fill_value = constants::DefaultFillValue().value; - // Check result: diag should be filled with var_fill_value auto some_field = qc.clone(); some_field.deep_copy(var_fill_value); REQUIRE(views_are_equal(diag_f, some_field)); - util::TimeStamp t1({2024, 1, 2}, {0, 0, 0}); // a day later? + some_field.deep_copy(qc); + + util::TimeStamp t1({2024, 1, 2}, {0, 0, 0}); // a day later + const Real a_day = 24.0 * 60.0 * 60.0; // seconds qc.get_header().get_tracking().update_time_stamp(t1); - qc_v(0, 0) = 29.0; - qc.sync_to_dev(); - // diag->initialize(t1, RunType::Initial); - // diag->update(t1, RunType::Initial); + randomize(qc, engine, pdf); + + // Run diag again diag->compute_diagnostic(); - diag_f = diag->get_diagnostic(); - auto diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 0) == 1.0 / 3600.0); + some_field.update(qc, 1.0 / a_day, -1.0 / a_day); + REQUIRE(views_are_equal(diag_f, some_field)); + + // This should fail (return false): + some_field.update(qc, 1.0 / a_day, -1.0 / a_day); + REQUIRE_FALSE(views_are_equal(diag_f, some_field)); } } // namespace scream From 5ecca368768512b95dc27868ba205a0d891eaed2 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 1 Jul 2024 16:19:17 -0700 Subject: [PATCH 056/477] rename the tend and use init_timestep --- .../eamxx/src/diagnostics/CMakeLists.txt | 2 +- .../{atm_tend.cpp => atm_backtend.cpp} | 42 +++++++++------ .../eamxx/src/diagnostics/atm_backtend.hpp | 53 +++++++++++++++++++ components/eamxx/src/diagnostics/atm_tend.hpp | 47 ---------------- .../src/diagnostics/register_diagnostics.hpp | 4 +- .../src/diagnostics/tests/CMakeLists.txt | 2 +- ...tm_tend_test.cpp => atm_backtend_test.cpp} | 14 ++--- .../eamxx/src/share/io/scorpio_output.cpp | 6 +-- 8 files changed, 92 insertions(+), 78 deletions(-) rename components/eamxx/src/diagnostics/{atm_tend.cpp => atm_backtend.cpp} (66%) create mode 100644 components/eamxx/src/diagnostics/atm_backtend.hpp delete mode 100644 components/eamxx/src/diagnostics/atm_tend.hpp rename components/eamxx/src/diagnostics/tests/{atm_tend_test.cpp => atm_backtend_test.cpp} (89%) diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index 75c6f420fc61..be51f4346155 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -20,7 +20,7 @@ set(DIAGNOSTIC_SRCS aodvis.cpp number_path.cpp aerocom_cld.cpp - atm_tend.cpp + atm_backtend.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/atm_tend.cpp b/components/eamxx/src/diagnostics/atm_backtend.cpp similarity index 66% rename from components/eamxx/src/diagnostics/atm_tend.cpp rename to components/eamxx/src/diagnostics/atm_backtend.cpp index 1420ea8f8478..7904d3aa797d 100644 --- a/components/eamxx/src/diagnostics/atm_tend.cpp +++ b/components/eamxx/src/diagnostics/atm_backtend.cpp @@ -1,4 +1,4 @@ -#include "diagnostics/atm_tend.hpp" +#include "diagnostics/atm_backtend.hpp" #include @@ -6,19 +6,19 @@ namespace scream { -AtmTendDiag::AtmTendDiag(const ekat::Comm &comm, - const ekat::ParameterList ¶ms) +AtmBackTendDiag::AtmBackTendDiag(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) : AtmosphereDiagnostic(comm, params) { - EKAT_REQUIRE_MSG(params.isParameter("Field Name"), - "Error! AtmTendDiag requires 'Field Name' in its " + EKAT_REQUIRE_MSG(params.isParameter("Tendency Name"), + "Error! AtmBackTendDiag requires 'Tendency Name' in its " "input parameters.\n"); - m_name = m_params.get("Field Name"); + m_name = m_params.get("Tendency Name"); } -std::string AtmTendDiag::name() const { return m_name + "_atm_tend"; } +std::string AtmBackTendDiag::name() const { return m_name + "_atm_tend"; } -void AtmTendDiag::set_grids( +void AtmBackTendDiag::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -26,7 +26,7 @@ void AtmTendDiag::set_grids( add_field(m_name, gname); } -void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { +void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { const auto &f = get_field_in(m_name); const auto &fid = f.get_header().get_identifier(); const auto &gn = fid.get_grid_name(); @@ -34,13 +34,14 @@ void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { // Sanity checks using namespace ShortFieldTagsNames; const auto &layout = fid.get_layout(); - EKAT_REQUIRE_MSG(f.data_type() == DataType::RealType, - "Error! AtmTendDiag only supports Real data type field.\n" - " - field name: " + - fid.name() + - "\n" - " - field data type: " + - e2str(f.data_type()) + "\n"); + EKAT_REQUIRE_MSG( + f.data_type() == DataType::RealType, + "Error! AtmBackTendDiag only supports Real data type field.\n" + " - field name: " + + fid.name() + + "\n" + " - field data type: " + + e2str(f.data_type()) + "\n"); using namespace ekat::units; // The units are the same except per second @@ -57,7 +58,14 @@ void AtmTendDiag::initialize_impl(const RunType /*run_type*/) { m_f_prev = Field(prev_fid); m_f_prev.allocate_view(); } -void AtmTendDiag::compute_diagnostic_impl() { + +void AtmBackTendDiag::init_timestep(const util::TimeStamp &start_of_step) { + m_start_t = start_of_step; + auto f_curr = get_field_in(m_name); + m_f_prev.deep_copy(f_curr); +} + +void AtmBackTendDiag::compute_diagnostic_impl() { Real var_fill_value = constants::DefaultFillValue().value; std::int64_t dt; diff --git a/components/eamxx/src/diagnostics/atm_backtend.hpp b/components/eamxx/src/diagnostics/atm_backtend.hpp new file mode 100644 index 000000000000..d4698d82a052 --- /dev/null +++ b/components/eamxx/src/diagnostics/atm_backtend.hpp @@ -0,0 +1,53 @@ +#ifndef EAMXX_ATM_BACKTEND_DIAG_HPP +#define EAMXX_ATM_BACKTEND_DIAG_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/util/scream_time_stamp.hpp" + +namespace scream { + +/* + * This diagnostic will back out the atmosphere tendency of a given field. + */ + +class AtmBackTendDiag : public AtmosphereDiagnostic { + public: + // Constructors + AtmBackTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const; + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + // Let's override the init time step method + void init_timestep(const util::TimeStamp &start_of_step) override; + + // Let's override the initialize method to set the fields below + void initialize_impl(const RunType /*run_type*/) override; + + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + // The tendency of what? + std::string m_name; + + // Store the previous field + Field m_f_prev; + + // Store the timestamp of the start of the timestep + util::TimeStamp m_start_t; + +}; // class AtmBackTendDiag + +} // namespace scream + +#endif // EAMXX_ATM_BACKTEND_DIAG_HPP diff --git a/components/eamxx/src/diagnostics/atm_tend.hpp b/components/eamxx/src/diagnostics/atm_tend.hpp deleted file mode 100644 index 0cc707e7d4a5..000000000000 --- a/components/eamxx/src/diagnostics/atm_tend.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef EAMXX_ATM_TEND_DIAG_HPP -#define EAMXX_ATM_TEND_DIAG_HPP - -#include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_time_stamp.hpp" - -namespace scream { - -/* - * This diagnostic will produce the atmospheric tendency. - */ - -class AtmTendDiag : public AtmosphereDiagnostic { - public: - // Constructors - AtmTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); - - // The name of the diagnostic - std::string name() const; - - // Set the grid - void set_grids(const std::shared_ptr grids_manager); - - protected: -#ifdef KOKKOS_ENABLE_CUDA - public: -#endif - void compute_diagnostic_impl(); - - protected: - void initialize_impl(const RunType /*run_type*/); - - // Keep track of field dimensions - int m_num_cols; - int m_num_levs; - - // The tendency of what? - std::string m_name; - - // Store the previous field - Field m_f_prev; - -}; // class AtmTendDiag - -} // namespace scream - -#endif // EAMXX_ATM_TEND_DIAG_HPP diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 07f90624db03..f22ec478be54 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -23,7 +23,7 @@ #include "diagnostics/aodvis.hpp" #include "diagnostics/number_path.hpp" #include "diagnostics/aerocom_cld.hpp" -#include "diagnostics/atm_tend.hpp" +#include "diagnostics/atm_backtend.hpp" namespace scream { @@ -54,7 +54,7 @@ inline void register_diagnostics () { diag_factory.register_product("AerosolOpticalDepth550nm",&create_atmosphere_diagnostic); diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); diag_factory.register_product("AeroComCld",&create_atmosphere_diagnostic); - diag_factory.register_product("AtmTendDiag",&create_atmosphere_diagnostic); + diag_factory.register_product("AtmBackTendDiag",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 1dfb92fa7e29..a684aa248fc0 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -72,6 +72,6 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") # Test atm_tend - CreateDiagTest(atm_tend "atm_tend_test.cpp") + CreateDiagTest(atm_backtend "atm_backtend_test.cpp") endif() diff --git a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp similarity index 89% rename from components/eamxx/src/diagnostics/tests/atm_tend_test.cpp rename to components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp index 79d1386bbaf8..e8e2b0c29c90 100644 --- a/components/eamxx/src/diagnostics/tests/atm_tend_test.cpp +++ b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp @@ -26,7 +26,7 @@ std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, return gm; } -TEST_CASE("atm_tend") { +TEST_CASE("atm_backtend") { using namespace ShortFieldTagsNames; using namespace ekat::units; @@ -37,8 +37,8 @@ TEST_CASE("atm_tend") { util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); // Create a grids manager - single column for these tests - constexpr int nlevs = 25; - const int ngcols = 25 * comm.size(); + constexpr int nlevs = 7; + const int ngcols = 2 * comm.size(); auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); @@ -62,8 +62,8 @@ TEST_CASE("atm_tend") { register_diagnostics(); ekat::ParameterList params; - REQUIRE_THROWS( - diag_factory.create("AtmTendDiag", comm, params)); // No 'Field Name' + REQUIRE_THROWS(diag_factory.create("AtmBackTendDiag", comm, + params)); // No 'Tendency Name' Real var_fill_value = constants::DefaultFillValue().value; @@ -73,8 +73,8 @@ TEST_CASE("atm_tend") { // Create and set up the diagnostic params.set("grid_name", grid->name()); - params.set("Field Name", "qc"); - auto diag = diag_factory.create("AtmTendDiag", comm, params); + params.set("Tendency Name", "qc"); + auto diag = diag_factory.create("AtmBackTendDiag", comm, params); diag->set_grids(gm); diag->set_required_field(qc); diag->initialize(t0, RunType::Initial); diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 8b7b861ff440..9c04a0d69d3b 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1377,13 +1377,13 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { diag_name = "VaporFlux"; // split will return the list [X, ''], with X being whatever is before 'VapFlux' params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); - } else if (diag_field_name.find("_atm_tend")!=std::string::npos) { - diag_name = "AtmTendDiag"; + } else if (diag_field_name.find("_atm_backtend")!=std::string::npos) { + diag_name = "AtmBackTendDiag"; // TODO: not sure if this is needed? Can skip, but what to do inside diag? // Set the grid_name params.set("grid_name",get_field_manager("sim")->get_grid()->name()); // split will return [X, ''], with X being whatever is before '_atm_tend' - params.set("Tend Name",ekat::split(diag_field_name,"_atm_tend").front()); + params.set("Tendency Name",ekat::split(diag_field_name,"_atm_backtend").front()); } else if (diag_field_name=="PotentialTemperature" or diag_field_name=="LiqPotentialTemperature") { diag_name = "PotentialTemperature"; From 86586bf98119316f9a45f6e5092a391a6d87e98e Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 1 Jul 2024 16:51:40 -0700 Subject: [PATCH 057/477] drop unneeded comments from code --- components/eamxx/src/diagnostics/atm_backtend.cpp | 5 ++--- components/eamxx/src/share/io/scorpio_output.cpp | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/diagnostics/atm_backtend.cpp b/components/eamxx/src/diagnostics/atm_backtend.cpp index 7904d3aa797d..c29bb665537a 100644 --- a/components/eamxx/src/diagnostics/atm_backtend.cpp +++ b/components/eamxx/src/diagnostics/atm_backtend.cpp @@ -46,7 +46,6 @@ void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { using namespace ekat::units; // The units are the same except per second auto diag_units = fid.get_units() / s; - // TODO: set the units string correctly by appending "/s" // All good, create the diag output FieldIdentifier d_fid(name(), layout.clone(), diag_units, gn); @@ -60,8 +59,8 @@ void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { } void AtmBackTendDiag::init_timestep(const util::TimeStamp &start_of_step) { - m_start_t = start_of_step; - auto f_curr = get_field_in(m_name); + m_start_t = start_of_step; + const auto &f_curr = get_field_in(m_name); m_f_prev.deep_copy(f_curr); } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 9c04a0d69d3b..811a5fb42c62 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1379,7 +1379,6 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); } else if (diag_field_name.find("_atm_backtend")!=std::string::npos) { diag_name = "AtmBackTendDiag"; - // TODO: not sure if this is needed? Can skip, but what to do inside diag? // Set the grid_name params.set("grid_name",get_field_manager("sim")->get_grid()->name()); // split will return [X, ''], with X being whatever is before '_atm_tend' From a1621df4fab561d262b0b0c592a1c69d43a0f9be Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 2 Jul 2024 08:07:22 -0700 Subject: [PATCH 058/477] better reorganization of logic --- .../eamxx/src/diagnostics/atm_backtend.cpp | 4 +-- .../eamxx/src/diagnostics/atm_backtend.hpp | 3 -- .../diagnostics/tests/atm_backtend_test.cpp | 32 +++++++++++-------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/diagnostics/atm_backtend.cpp b/components/eamxx/src/diagnostics/atm_backtend.cpp index c29bb665537a..4b147de2d6c8 100644 --- a/components/eamxx/src/diagnostics/atm_backtend.cpp +++ b/components/eamxx/src/diagnostics/atm_backtend.cpp @@ -59,9 +59,9 @@ void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { } void AtmBackTendDiag::init_timestep(const util::TimeStamp &start_of_step) { - m_start_t = start_of_step; const auto &f_curr = get_field_in(m_name); m_f_prev.deep_copy(f_curr); + m_f_prev.get_header().get_tracking().update_time_stamp(start_of_step); } void AtmBackTendDiag::compute_diagnostic_impl() { @@ -83,8 +83,6 @@ void AtmBackTendDiag::compute_diagnostic_impl() { // yet, so fill with an invalid value m_diagnostic_output.deep_copy(var_fill_value); } - m_f_prev.deep_copy(f); - m_f_prev.get_header().get_tracking().update_time_stamp(curr_ts); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/atm_backtend.hpp b/components/eamxx/src/diagnostics/atm_backtend.hpp index d4698d82a052..a8bd72883dbe 100644 --- a/components/eamxx/src/diagnostics/atm_backtend.hpp +++ b/components/eamxx/src/diagnostics/atm_backtend.hpp @@ -43,9 +43,6 @@ class AtmBackTendDiag : public AtmosphereDiagnostic { // Store the previous field Field m_f_prev; - // Store the timestamp of the start of the timestep - util::TimeStamp m_start_t; - }; // class AtmBackTendDiag } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp index e8e2b0c29c90..47086b5ddedc 100644 --- a/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp +++ b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp @@ -43,7 +43,7 @@ TEST_CASE("atm_backtend") { auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); - // Input (randomized) qc, nc + // Input (randomized) qc FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; FieldIdentifier qc_fid("qc", scalar2d_layout, kg / kg, grid->name()); @@ -88,21 +88,27 @@ TEST_CASE("atm_backtend") { some_field.deep_copy(var_fill_value); REQUIRE(views_are_equal(diag_f, some_field)); - some_field.deep_copy(qc); + const Real a_day = 24.0 * 60.0 * 60.0; // seconds - util::TimeStamp t1({2024, 1, 2}, {0, 0, 0}); // a day later - const Real a_day = 24.0 * 60.0 * 60.0; // seconds - qc.get_header().get_tracking().update_time_stamp(t1); - randomize(qc, engine, pdf); + constexpr int ntests = 10; - // Run diag again - diag->compute_diagnostic(); - some_field.update(qc, 1.0 / a_day, -1.0 / a_day); - REQUIRE(views_are_equal(diag_f, some_field)); + for(int itest = 2; itest < ntests; itest++) { + // Run diag again + some_field.deep_copy(qc); + + diag->init_timestep(t0); + + util::TimeStamp t1({2024, 1, itest}, {0, 0, 0}); // a day later + qc.get_header().get_tracking().update_time_stamp(t1); + randomize(qc, engine, pdf); + + diag->compute_diagnostic(); + some_field.update(qc, 1.0 / a_day, -1.0 / a_day); + REQUIRE(views_are_equal(diag_f, some_field)); - // This should fail (return false): - some_field.update(qc, 1.0 / a_day, -1.0 / a_day); - REQUIRE_FALSE(views_are_equal(diag_f, some_field)); + // reset t0 to t1 to keep iterating... + t0 = t1; + } } } // namespace scream From a598acd9e9fdd7df5f0a4a4ff5f2e3ee7dc64956 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 09:55:14 -0700 Subject: [PATCH 059/477] Adds description for top_lev and adds its value for 128 levs --- components/eamxx/cime_config/namelist_defaults_scream.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50019833d8ae..50188236e4e0 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -237,7 +237,8 @@ be lost if SCREAM_HACK_XML is not enabled. 0.001 - 6 + 6 + 0 From c889de02c584c90cee76fae26be5572e58b94599 Mon Sep 17 00:00:00 2001 From: dqwu Date: Tue, 2 Jul 2024 16:21:06 -0500 Subject: [PATCH 060/477] Fixing build error in anlgce.cmake (unknown EKAT_MPIRUN_EXE) Move two include statements to the end of anlgce.cmake to fix the build error "EKAT_MPIRUN_EXE is not known". --- components/eamxx/cmake/machine-files/anlgce.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index 079000059d19..e79f70015471 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) - # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) From c533709f4ff112434dc1cef5a4336ad664dbcc73 Mon Sep 17 00:00:00 2001 From: dqwu Date: Wed, 29 May 2024 12:11:12 -0500 Subject: [PATCH 061/477] Adding cmake file for ANL UB22 workstation Adding cmake file for building SCREAM on ANL UB22 workstation --- .../eamxx/cmake/machine-files/anlgce-ub22.cmake | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 components/eamxx/cmake/machine-files/anlgce-ub22.cmake diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake new file mode 100644 index 000000000000..e79f70015471 --- /dev/null +++ b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake @@ -0,0 +1,12 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Remove this if you are using a resource manager (slurm etc) +set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") + +# EKAT MPI settings +set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) From a9f16f1da78ac591faf91848de499b297a9c3a1b Mon Sep 17 00:00:00 2001 From: dqwu Date: Wed, 29 May 2024 13:33:50 -0500 Subject: [PATCH 062/477] Support for EAMXX/SCREAM testing in ANL GCE (UB22) Adding support for EAMXX standalone testing on ANL GCE (UB22) machine (Run EAMXX standalone tests on ANL GCE (UB22) using "./scripts/test-all-scream -m anlgce-ub22 ...") --- components/eamxx/scripts/machines_specs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 9536d415c0de..43e5ebfa083c 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,6 +82,10 @@ ["mpicxx","mpifort","mpicc"], "", ""), + "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), From 82ee85e25b93a06aae23765e6d178e608b9b7303 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 2 Jul 2024 13:27:34 -0600 Subject: [PATCH 063/477] Add P3 vars for MAM4xx --- .../src/physics/p3/disp/p3_main_impl_disp.cpp | 6 ++++-- .../src/physics/p3/eamxx_p3_process_interface.cpp | 14 +++++++++----- .../eamxx/src/physics/p3/impl/p3_main_impl.hpp | 14 ++++++++------ components/eamxx/src/physics/p3/p3_functions.hpp | 4 ++++ .../eamxx/src/physics/p3/p3_functions_f90.cpp | 2 +- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index c3692f3f814f..d20350b0788b 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -162,8 +162,8 @@ ::p3_main_internal_disp( qtend_ignore("qtend_ignore", nj, nk_pack), ntend_ignore("ntend_ignore", nj, nk_pack), // Variables still used in F90 but removed from C++ interface - mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), precip_total_tend("precip_total_tend", nj, nk_pack), - nevapr("nevapr", nj, nk_pack), qr_evap_tend("qr_evap_tend", nj, nk_pack), + mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), + qr_evap_tend("qr_evap_tend", nj, nk_pack), // cloud sedimentation v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), @@ -205,6 +205,8 @@ ::p3_main_internal_disp( auto rho_qi = diagnostic_outputs.rho_qi; auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; + auto precip_total_tend = diagnostic_outputs.precip_total_tend; + auto nevapr = diagnostic_outputs.nevapr; auto qv_prev = diagnostic_inputs.qv_prev; auto t_prev = diagnostic_inputs.t_prev; auto liq_ice_exchange = history_only.liq_ice_exchange; diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index d886478ac2af..acaecd7eef70 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -95,11 +95,13 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m add_field ("T_prev_micro_step", scalar3d_layout_mid, K, grid_name, ps); // Diagnostic Outputs: (all fields are just outputs w.r.t. P3) - add_field("precip_liq_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); - add_field("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); - add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name, ps); - add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name, ps); - add_field("eff_radius_qr", scalar3d_layout_mid, micron, grid_name, ps); + add_field("precip_liq_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); + add_field("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); + add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name, ps); + add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name, ps); + add_field("eff_radius_qr", scalar3d_layout_mid, micron, grid_name, ps); + add_field("precip_total_tend", scalar3d_layout_mid, kg/(kg*s), grid_name, ps); + add_field("nevapr", scalar3d_layout_mid, kg/(kg*s), grid_name, ps); // History Only: (all fields are just outputs and are really only meant for I/O purposes) // TODO: These should be averaged over subcycle as well. But there is no simple mechanism @@ -307,6 +309,8 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) diag_outputs.diag_eff_radius_qc = get_field_out("eff_radius_qc").get_view(); diag_outputs.diag_eff_radius_qi = get_field_out("eff_radius_qi").get_view(); diag_outputs.diag_eff_radius_qr = get_field_out("eff_radius_qr").get_view(); + diag_outputs.precip_total_tend = get_field_out("precip_total_tend").get_view(); + diag_outputs.nevapr = get_field_out("nevapr").get_view(); diag_outputs.precip_liq_surf = m_buffer.precip_liq_surf_flux; diag_outputs.precip_ice_surf = m_buffer.precip_ice_surf_flux; diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index b05b713aec29..ce5116cab2fa 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -142,9 +142,9 @@ ::p3_main_internal( qtend_ignore, ntend_ignore, // Variables still used in F90 but removed from C++ interface - mu_c, lamc, precip_total_tend, nevapr, qr_evap_tend; + mu_c, lamc, qr_evap_tend; - workspace.template take_many_and_reset<46>( + workspace.template take_many_and_reset<44>( { "mu_r", "T_atm", "lamr", "logn0r", "nu", "cdist", "cdist1", "cdistr", "inv_cld_frac_i", "inv_cld_frac_l", "inv_cld_frac_r", "qc_incld", "qr_incld", "qi_incld", "qm_incld", @@ -153,7 +153,7 @@ ::p3_main_internal( "rhofacr", "rhofaci", "acn", "qv_sat_l", "qv_sat_i", "sup", "qv_supersat_i", "tmparr1", "exner", "diag_equiv_reflectivity", "diag_vm_qi", "diag_diam_qi", "pratot", "prctot", "qtend_ignore", "ntend_ignore", - "mu_c", "lamc", "precip_total_tend", "nevapr", "qr_evap_tend" + "mu_c", "lamc", "qr_evap_tend" }, { &mu_r, &T_atm, &lamr, &logn0r, &nu, &cdist, &cdist1, &cdistr, @@ -163,7 +163,7 @@ ::p3_main_internal( &rhofacr, &rhofaci, &acn, &qv_sat_l, &qv_sat_i, &sup, &qv_supersat_i, &tmparr1, &exner, &diag_equiv_reflectivity, &diag_vm_qi, &diag_diam_qi, &pratot, &prctot, &qtend_ignore, &ntend_ignore, - &mu_c, &lamc, &precip_total_tend, &nevapr, &qr_evap_tend + &mu_c, &lamc, &qr_evap_tend }); // Get single-column subviews of all inputs, shouldn't need any i-indexing @@ -197,6 +197,8 @@ ::p3_main_internal( const auto orho_qi = ekat::subview(diagnostic_outputs.rho_qi, i); const auto oprecip_liq_flux = ekat::subview(diagnostic_outputs.precip_liq_flux, i); const auto oprecip_ice_flux = ekat::subview(diagnostic_outputs.precip_ice_flux, i); + const auto oprecip_total_tend = ekat::subview(diagnostic_outputs.precip_total_tend, i); + const auto onevapr = ekat::subview(diagnostic_outputs.nevapr, i); const auto oliq_ice_exchange = ekat::subview(history_only.liq_ice_exchange, i); const auto ovap_liq_exchange = ekat::subview(history_only.vap_liq_exchange, i); const auto ovap_ice_exchange = ekat::subview(history_only.vap_ice_exchange, i); @@ -216,7 +218,7 @@ ::p3_main_internal( &nc_incld, &nr_incld, &ni_incld, &bm_incld, &inv_rho, &prec, &rho, &rhofacr, &rhofaci, &acn, &qv_sat_l, &qv_sat_i, &sup, &qv_supersat_i, &tmparr1, &qtend_ignore, &ntend_ignore, - &mu_c, &lamc, &orho_qi, &oqv2qi_depos_tend, &precip_total_tend, &nevapr, &oprecip_liq_flux, &oprecip_ice_flux + &mu_c, &lamc, &orho_qi, &oqv2qi_depos_tend, &oprecip_total_tend, &onevapr, &oprecip_liq_flux, &oprecip_ice_flux }; // initialize @@ -252,7 +254,7 @@ ::p3_main_internal( oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, olatent_heat_vapor, olatent_heat_sublim, olatent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, - mu_r, lamr, logn0r, oqv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, + mu_r, lamr, logn0r, oqv2qi_depos_tend, oprecip_total_tend, onevapr, qr_evap_tend, ovap_liq_exchange, ovap_ice_exchange, oliq_ice_exchange, pratot, prctot, hydrometeorsPresent, nk, p3constants); diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 1749a4ae7346..47bb4560c2e1 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -191,6 +191,10 @@ struct Functions view_2d precip_liq_flux; // Grid-box average ice/snow flux [kg m^-2 s^-1] pverp view_2d precip_ice_flux; + // Total precipitation (rain + snow) [kg/kg/s] + view_2d precip_total_tend; + // Evaporation of total precipitation (rain + snow) [kg/kg/s] + view_2d nevapr; }; // This struct stores time stepping and grid-index-related information. diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 83aeb5b16d3d..eff3102f0e44 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -2048,7 +2048,7 @@ Int p3_main_f( inv_exner_d, qv_prev_d, t_prev_d}; P3F::P3DiagnosticOutputs diag_outputs{qv2qi_depos_tend_d, precip_liq_surf_d, precip_ice_surf_d, diag_eff_radius_qc_d, diag_eff_radius_qi_d, diag_eff_radius_qr_d, - rho_qi_d,precip_liq_flux_d, precip_ice_flux_d}; + rho_qi_d,precip_liq_flux_d, precip_ice_flux_d, precip_total_tend_d, nevapr_d}; P3F::P3Infrastructure infrastructure{dt, it, its, ite, kts, kte, do_predict_nc, do_prescribed_CCN, col_location_d}; P3F::P3HistoryOnly history_only{liq_ice_exchange_d, vap_liq_exchange_d, From 778b6ac1b1221f83ecf1fbaf67827a523460442f Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Fri, 5 Jul 2024 13:59:21 -0600 Subject: [PATCH 064/477] Remove add_field() calls that are not needed by the Dry Deposition process. --- ...x_mam_dry_deposition_process_interface.cpp | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 5ac7489f1400..2968b9927ef5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -96,7 +96,6 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, @@ -115,22 +114,23 @@ void MAMDryDep::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - // add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); - add_field("obklen", scalar2d_layout, m, grid_name); - add_field("surfric", scalar2d_layout, m / s, grid_name); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - add_field("icefrac", scalar2d_layout, nondim, grid_name); - add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - add_field("fv", scalar2d_layout, m / s, grid_name); - add_field("ram1", scalar2d_layout, s / m, grid_name); + // TODO: The following are not used by drydep but to create a dry atmosphere object. + add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction + // + // TODO: Currently the following are scalar parameters that are fixed + // for the whole grid. Do these need to be column specific? + // add_field("obklen", scalar2d_layout, m, grid_name); + // add_field("surfric", scalar2d_layout, m / s, grid_name); + // add_field("landfrac", scalar2d_layout, nondim, grid_name); + // add_field("icefrac", scalar2d_layout, nondim, grid_name); + // add_field("ocnfrac", scalar2d_layout, nondim, grid_name); + // add_field("fv", scalar2d_layout, m / s, grid_name); + // add_field("ram1", scalar2d_layout, s / m, grid_name); // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -407,10 +407,7 @@ void fill_tracer_views( // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - using DryDep = mam4::DryDeposition; - - using MemberType = KT::MemberType; const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); From 636c7654d2fab21b3a7f19eede1a1f9106ff81b7 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Mar 2024 08:52:57 -0600 Subject: [PATCH 065/477] wet_scav - get code from balwinder branch. --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- .../eamxx_mam_wetscav_process_interface.cpp | 874 ++++++++++++++++++ .../eamxx_mam_wetscav_process_interface.hpp | 210 +++++ .../uncoupled/mam4_wet_scav/CMakeLists.txt | 30 + .../tests/uncoupled/mam4_wet_scav/input.yaml | 68 ++ .../mam4_wet_scav/mam4_wetscav_standalone.cpp | 69 ++ .../tests/uncoupled/mam4_wet_scav/output.yaml | 14 + 7 files changed, 1267 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp create mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt create mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml create mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp create mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/output.yaml diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 40f43cbf3a7e..dd8cf6e6b304 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -43,7 +43,8 @@ add_subdirectory(${EXTERNALS_SOURCE_DIR}/mam4xx ${CMAKE_BINARY_DIR}/externals/ma add_library(mam eamxx_mam_microphysics_process_interface.cpp eamxx_mam_optics_process_interface.cpp - eamxx_mam_aci_process_interface.cpp) + eamxx_mam_aci_process_interface.cpp + eamxx_mam_microphysics_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp new file mode 100644 index 000000000000..7229419b378a --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -0,0 +1,874 @@ +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +// NOTE: see the impl/ directory for the contents of the impl namespace +#include "impl/compute_particle_size.cpp" + +// Remove the following<<<< +#include +#include +//>>>>>>> + +/* +Future work: +Wirte comments +write in/outs for all variables clearly + + +*/ + +namespace scream { + +// ========================================================================================= +MAMWetscav::MAMWetscav(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ========================================================================================= +void MAMWetscav::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + + m_grid = grids_manager->get_grid("Physics"); + const auto &grid_name = m_grid->name(); + + ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank + nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + + // Layout for 2D (2d horiz) variable defined at mid-level and + // interfaces + const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + + // -- Input variables that exists in PBUF in EAM + static constexpr auto nondim = + Units::nondimensional(); // for variables that are fractions etc. + + // MUST FIXME: cldt and cldn are the same variables. They must be their + // previous step values. + add_field("cldn_prev_step", scalar3d_layout_mid, nondim, + grid_name); // layer cloud fraction [fraction] + add_field("cldst", scalar3d_layout_mid, nondim, + grid_name); //?? + add_field("rprdsh", scalar3d_layout_mid, kg / kg / s, + grid_name); // rain production, shallow convection + // [kg/kg/s] //NOT updated + add_field( + "rprddp", scalar3d_layout_mid, kg / kg / s, + grid_name); // rain production, deep convection [kg/kg/s] //NOT updated + add_field("evapcsh", scalar3d_layout_mid, kg / kg / s, + grid_name); // Evaporation rate of shallow convective + // //NOT updated precipitation >=0. [kg/kg/s] + add_field("evapcdp", scalar3d_layout_mid, kg / kg / s, + grid_name); // Evaporation rate of deep convective //NOT + // updated precipitation >=0. [kg/kg/s] + + // -- Input variables that exists in PBUF in EAM (in wetdep.F90) in the + // "inputs" data structure + // MUST FIXME: cldt and cldn are the same variables. They must be their + // previous step values. + add_field("cldt_prev_step", scalar3d_layout_mid, nondim, + grid_name); // total cloud fraction [fraction] + add_field( + "qme", scalar3d_layout_mid, kg / kg / s, + grid_name); // net condensation/evaporation of cloud water [kg/kg/s] + add_field("prain", scalar3d_layout_mid, kg / kg / s, + grid_name); // stratiform rain production rate [kg/kg/s] + add_field( + "evapr", scalar3d_layout_mid, kg / kg / s, + grid_name); // evaporation from stratiform rain [kg/kg/s] //NOT updated + + // -- Input variables that exists in PBUF in EAM (in wetdep.F90) + add_field("icwmrdp", scalar3d_layout_mid, kg / kg, + grid_name); // In cloud water mixing ratio, deep + // convection [kg/kg] //NOT updated + add_field("icwmrsh", scalar3d_layout_mid, kg / kg, + grid_name); // In cloud water mixing ratio, shallow + // convection [kg/kg] //NOT updated + add_field("rprddp", scalar3d_layout_mid, kg / kg / s, + grid_name); // Rain production, deep convection [kg/kg/s] + add_field( + "sh_frac", scalar3d_layout_mid, nondim, + grid_name); // Shallow convective cloud fraction [fraction] //NOT updated + add_field( + "dp_frac", scalar3d_layout_mid, nondim, + grid_name); // Deep convective cloud fraction [fraction] //NOT updated + add_field( + "icwmrsh", scalar3d_layout_mid, nondim, + grid_name); + add_field( + "icwmrdp", scalar3d_layout_mid, nondim, + grid_name); + add_field("evapcsh", scalar3d_layout_mid, kg / kg / s, + grid_name); // Evaporation rate of shallow convective + // precipitation >=0. [kg/kg/s] + add_field("evapcdp", scalar3d_layout_mid, kg / kg / s, + grid_name); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + + // -- surface fluxes (input/outpts) for the coupler's cam_out data struture + // for the land model + static constexpr auto m2 = m * m; + add_field( + "bcphiwet", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of hydrophilic black carbon [kg/m2/s] + add_field( + "bcphidry", scalar3d_layout_mid, kg / m2 / s, + grid_name); // dry deposition of hydrophilic black carbon [kg/m2/s] + add_field( + "ocphiwet", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of hydrophilic organic carbon [kg/m2/s] + add_field( + "ocphidry", scalar3d_layout_mid, kg / m2 / s, + grid_name); // dry deposition of hydrophilic organic carbon [kg/m2/s] + + add_field("dstwet1", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of dust (bin1) [kg/m2/s] + add_field("dstwet2", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of dust (bin2) [kg/m2/s] + add_field("dstwet3", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of dust (bin3) [kg/m2/s] + add_field("dstwet4", scalar3d_layout_mid, kg / m2 / s, + grid_name); // wet deposition of dust (bin4) [kg/m2/s] + + // -- input/ouputs from PBUF for updating particle size and water uptake by + // particles + static constexpr auto m3 = m2 * m; + add_field("dgncur_a", scalar3d_layout_mid, m, + grid_name); // aerosol particle diameter [m] + add_field("wetdens", scalar3d_layout_mid, kg / m3, + grid_name); // wet aerosol density [kg/m3] + add_field("qaerwat", scalar3d_layout_mid, kg / kg, + grid_name); // aerosol water [kg/kg] + add_field("dgnumwet", scalar3d_layout_mid, m, + grid_name); // wet aerosol diameter [m] + add_field("fracis", scalar3d_layout_mid, nondim, + grid_name); // fraction of transported species that are + // insoluble [fraction] + + // -- interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + // -- NOTE: Interstitial aerosols are updated in the interface using the + // "tendencies" from the wetscavenging process + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(imode); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(imode); + + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(imode, ispec); + if(strlen(cld_mmr_field_name) > 0) { + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these + // are NOT advected + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // Tracers group -- do we need this in addition to the tracers above? In any + // case, this call should be idempotent, so it can't hurt. + add_group("tracers", grid_name, 1, Bundling::Required); + + // The following fields are not needed by this process but we define them so + // that we can create MAM4xx class objects like atmosphere, prognostics etc. + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + grid_name); // Cloud fraction + add_field("pbl_height", scalar2d_layout_mid, m, + grid_name); // PBL height + + static constexpr auto s2 = s * s; + add_field("phis", scalar2d_layout_mid, m2 / s2, + grid_name); // surface geopotential + add_field("qv", scalar3d_layout_mid, q_unit, + grid_name); // specific humidity + add_field("nc", scalar3d_layout_mid, n_unit, + grid_name); // cloud water number conc + add_field("ni", scalar3d_layout_mid, n_unit, + grid_name); // cloud ice number conc + add_field("omega", scalar3d_layout_mid, Pa / s, + grid_name); // vertical pressure velocity +} + +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMWetscav::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMWetscav::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMWetscav."); +} + +// ========================================================================================= +void MAMWetscav::initialize_impl(const RunType run_type) { + // Gather runtime options + //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + // populate the wet atmosphere state with views from fields and + // the buffer (NOTE: wet atmosphere only has qv, qc, qi, nc, ni and omega) + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + + // -- Following wet atm variables are NOT used by the process but we still + // need them to + // -- create atmosphere object + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + // populate the dry atmosphere state with views from fields + // (NOTE: dry atmosphere has everything that wet + // atmosphere has along with z_surf, T_mid, p_mid, z_mid, z_iface, + // dz, p_del, cldfrac, w_updraft, pblh, phis) + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + + // How "buffer_" works: We use buffer to allocate memory for the members of + // dry_atm_ object. Here we are providing those memory locations to the + // dry_atm_ members. These members are computed from the above wet_atm_ or + // dry_atm_ members that are explicitly getting their values either from the + // input file or from other processes. These members are null at this point, + // they are assigned in "Kokkos::parallel_for("preprocess", scan_policy, + // preprocess_);" call in the run_impl + + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.dz = buffer_.dz; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.w_updraft = buffer_.w_updraft; + + // The following dry_atm_ members *may* not be used by the process but they + // are needed for creating MAM4xx class objects like Atmosphere + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.z_surf = 0.0; // MUST FIXME: for now + // ---- set wet/dry aerosol-related gas state data + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // Other required variables + cldn_prev_step_ = get_field_out("cldn_prev_step").get_view< Real **>(); + cldt_prev_step_ = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + cldst_ = get_field_out("cldst").get_view< Real **>();//?? + evapr_ = get_field_out("evapr").get_view< Real **>(); + rprdsh_ = + get_field_out("rprdsh").get_view(); // rain production, shallow + // convection [kg/kg/s] + evapcsh_ = + get_field_out("evapcsh") + .get_view(); // Evaporation rate of shallow convective + // precipitation >=0. [kg/kg/s] + + sh_frac_ = get_field_out("sh_frac") + .get_view(); // Shallow convective cloud fraction [fraction] + + rprddp_ = + get_field_out("rprddp") + .get_view(); // rain production, deep convection [kg/kg/s] + + evapcdp_ = get_field_out("evapcdp") + .get_view(); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + dp_frac_ = get_field_out("dp_frac") + .get_view(); // Deep convective cloud fraction [fraction] + + icwmrsh_ = get_field_out("icwmrsh") + .get_view(); // ?? + + icwmrdp_ = get_field_out("icwmrdp") + .get_view(); // ?? + + // set wet/dry aerosol state data (interstitial aerosols only) + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(imode); + wet_aero_.int_aero_nmr[imode] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[imode] = buffer_.dry_int_aero_nmr[imode]; + + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(imode); + wet_aero_.cld_aero_nmr[imode] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[imode] = wet_aero_.cld_aero_nmr[imode]; + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[imode][ispec] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[imode][ispec] = + buffer_.dry_int_aero_mmr[imode][ispec]; + } + + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(imode, ispec); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[imode][ispec] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[imode][ispec] = + buffer_.dry_cld_aero_mmr[imode][ispec]; + } + } + } + + // set up our preprocess/postprocess functors + // Here we initialize (not compute) objects in preprocess struct using the + // objects in the argument list + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); +} + +// ========================================================================================= +void MAMWetscav::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of all variables + // needed by this process or setting up MAM4xx classes and their objects + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + /*Fortran code: + call modal_aero_calcsize_sub(state%ncol, state%lchnk, state%q, state%pdel, + dt, & !in qqcw, ptend, dgnumdry_m=dgncur_a) !inout + + ----subroutine modal_aero_calcsize_sub(ncol, lchnk, state_q, pdel, deltat, + qqcw, ptend, do_adjust_in, & do_aitacc_transfer_in, list_idx_in, + update_mmr_in, dgnumdry_m) */ + // mam4::CalcSizeProcess process_(aero_config_); //initiate MAM4xx calcsize + // process + + /* ---------------------------------------------------------------------------------------- + * Compute particle size using the calcsize process + * ---------------------------------------------------------------------------------------- + */ + + // -- configure the process + /* + * -- NOTES: 1. Flags for the inter-mode particle transfer + * (do_aitacc_transfer) and size adjustment (do_adjust) are TRUE by default + * a. Size adjustment is only done by changing aerosol + * numbers in the modes. + * 2. Interstitial and cld borne aerosols (i.e. "tends") mmr will + * be updated (update_mmr is TRUE by default) + */ + // MUST: FIXME: Make sure state is not updated...only tendencies should be + // updated!! + static constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; + int nspec_amode[ntot_amode_]; + int lspectype_amode[maxd_aspectype][ntot_amode_]; + mam4::Real specdens_amode[maxd_aspectype]; + int lmassptr_amode[maxd_aspectype][ntot_amode_]; + Real spechygro[maxd_aspectype]; + + int numptr_amode[ntot_amode_]; + int mam_idx[ntot_amode_][mam4::ndrop::nspec_max]; + int mam_cnst_idx[ntot_amode_][mam4::ndrop::nspec_max]; + + mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, + numptr_amode, specdens_amode, spechygro, + mam_idx, mam_cnst_idx); + + Real inv_density[ntot_amode_][mam4::AeroConfig::num_aerosol_ids()] = {}; + Real num2vol_ratio_min[ntot_amode_] = {}; + Real num2vol_ratio_max[ntot_amode_] = {}; + Real num2vol_ratio_max_nmodes[ntot_amode_] = {}; + Real num2vol_ratio_min_nmodes[ntot_amode_] = {}; + Real num2vol_ratio_nom_nmodes[ntot_amode_] = {}; + Real dgnmin_nmodes[ntot_amode_] = {}; + Real dgnmax_nmodes[ntot_amode_] = {}; + Real dgnnom_nmodes[ntot_amode_] = {}; + Real mean_std_dev_nmodes[ntot_amode_] = {}; + + bool noxf_acc2ait[mam4::AeroConfig::num_aerosol_ids()] = {}; + int n_common_species_ait_accum = {}; + int ait_spec_in_acc[mam4::AeroConfig::num_aerosol_ids()] = {}; + int acc_spec_in_ait[mam4::AeroConfig::num_aerosol_ids()] = {}; + mam4::modal_aero_calcsize::init_calcsize( + inv_density, num2vol_ratio_min, num2vol_ratio_max, + num2vol_ratio_max_nmodes, num2vol_ratio_min_nmodes, + num2vol_ratio_nom_nmodes, dgnmin_nmodes, dgnmax_nmodes, dgnnom_nmodes, + mean_std_dev_nmodes, noxf_acc2ait, n_common_species_ait_accum, + ait_spec_in_acc, acc_spec_in_ait); + + // diagnostics for visible band summed over modes + + // Note: Need to compute inv density using indexing from e3sm + for(int imode = 0; imode < ntot_amode_; ++imode) { + const int nspec = nspec_amode[imode]; + for(int isp = 0; isp < nspec; ++isp) { + const int idx = lspectype_amode[isp][imode] - 1; + inv_density[imode][isp] = 1.0 / specdens_amode[idx]; + } // isp + } // imode + + view_2d state_q( + "state_q", nlev_, + nvars_); // MUST FIXME: make it is 3d view to avoid race condition + view_2d qqcw("qqcw", nlev_, nvars_); + + + //create a const column view of zeros + view_1d zeros_nlev("zeros_nlev", nlev_); + Kokkos::deep_copy(zeros_nlev,0); + const_view_1d const_zeros_nlev(zeros_nlev); + + mam4::AeroConfig aero_config; + mam4::WetDeposition::Config wetdep_config;// = wetdep_.Config(); + wetdep_.init(aero_config,wetdep_config);//FIXME: Should we call this in the initialize???? + + + + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + + // loop over atmosphere columns and compute aerosol particle size + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const ThreadTeam &team) { + const int icol = team.league_rank(); // column index*/ + + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0, nlev_), [&](int klev) { + view_1d state_q_k = ekat::subview( + state_q, klev); // 1d view of size (nvars_) for storing all + // gasses and aerosols + mam_coupling::state_q_for_column_at_one_lev(dry_aero_, icol, klev, + state_q_k); + + view_1d qqcw_k = + ekat::subview(qqcw, klev); // 1d view of size (nvars_) for + // storing all gasses and aerosols + mam_coupling::qqcw_for_column_at_one_lev(dry_aero_, icol, klev, + qqcw_k); + + static constexpr bool do_adjust = true, do_aitacc_transfer = true, + update_mmr = true; + + Real dgncur_c_kk[ntot_amode_] = {}; + Real dgnumdry_m_kk[ntot_amode_] = {}; + // Calculate aerosol size distribution parameters and aerosol + // water + // uptake + mam4::modal_aero_calcsize::modal_aero_calcsize_sub( + state_q_k.data(), // in + qqcw_k.data(), // in/out + dt, do_adjust, do_aitacc_transfer, update_mmr, lmassptr_amode, + numptr_amode, + inv_density, // in + num2vol_ratio_min, num2vol_ratio_max, + num2vol_ratio_max_nmodes, num2vol_ratio_min_nmodes, + num2vol_ratio_nom_nmodes, dgnmin_nmodes, dgnmax_nmodes, + dgnnom_nmodes, mean_std_dev_nmodes, + // outputs + noxf_acc2ait, n_common_species_ait_accum, ait_spec_in_acc, + acc_spec_in_ait, dgnumdry_m_kk, dgncur_c_kk); + + Real dgnumwet_m_kk[ntot_amode_] = {}; + Real qaerwat_m_kk[ntot_amode_] = {}; + // std::cout< qqcw_get_field(pbuf,mm,lchnk) + qqcw_sav(1:ncol,:,lspec) = fldcw(1:ncol,:) !RCE 2012/01/12 + + ! FIXME: Not sure if this is a bug or not as qqcw_tmp seem + different ! from the previous call and qqcw_tmp is always zero. May need ! + further check. - Shuaiqi Tang in refactoring for MAM4xx call wetdepa_v2( & + ncol, dt, state%pdel, & ! in dep_inputs%cmfdqr, + dep_inputs%evapc, dlf, dep_inputs%conicw, & ! in dep_inputs%prain, + dep_inputs%evapr, dep_inputs%totcond, & ! in dep_inputs%cldt, + dep_inputs%cldcu, & ! in dep_inputs%cldvcu, + dep_inputs%cldvst, & ! in sol_factb, sol_facti, + sol_factic, & ! in mam_prevap_resusp_optcc, + .true., scavcoefnv(:,:,jnv), f_act_conv, & ! in fldcw, qqcw_tmp, & ! in + fracis_cw, dqdt_tmp, iscavt, & ! out icscavt, isscavt, + bcscavt, bsscavt, rcscavt, rsscavt ) ! out + + ! resuspension goes to coarse mode + call calc_resusp_to_coarse( ncol, mm, & ! in + mmtoo_prevap_resusp, .false., & ! in + rcscavt, rsscavt, & ! in + dqdt_tmp, rtscavt_sv ) ! inout + + fldcw(1:ncol,:) = fldcw(1:ncol,:) + dqdt_tmp(1:ncol,:) * dt + + call calc_sfc_flux(dqdt_tmp(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFWET', + sflx, pcols, lchnk) aerdepwetcw(:ncol,mm) = sflx(:ncol) + + call calc_sfc_flux(icscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSIC', + sflx, pcols, lchnk) + + call calc_sfc_flux(isscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSIS', + sflx, pcols, lchnk) + + call calc_sfc_flux(bcscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSBC', + sflx, pcols, lchnk) + + call calc_sfc_flux(bsscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSBS', + sflx, pcols, lchnk) + + call calc_sfc_flux(rcscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSEC', + sflx, pcols, lchnk) + + call calc_sfc_flux(rsscavt(:ncol,:), state%pdel(:ncol,:), + sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSES', + sflx, pcols, lchnk) + + endif lphase_jnmw_conditional + + enddo lspec_loop_aa ! lspec = 1, nspec_amode(m)+2 + enddo lphase_loop_aa ! lphase = 1, 2 + enddo mmode_loop_aa ! m = 1, ntot_amode + + ! if the user has specified prescribed aerosol dep fluxes then + ! do not set cam_out dep fluxes according to the prognostic aerosols + if (.not.aerodep_flx_prescribed()) then + call set_srf_wetdep(aerdepwetis, aerdepwetcw, cam_out) + endif + + call wetdep_inputs_unset(dep_inputs) + */ + std::cout << "End of wetscav run" << std::endl; +} + +// ========================================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp new file mode 100644 index 000000000000..bec616e260a1 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -0,0 +1,210 @@ +#ifndef EAMXX_MAM_WETSCAV_HPP +#define EAMXX_MAM_WETSCAV_HPP + +// For MAM4 aerosol configuration +#include + +// For declaring wetscav class derived from atm process class +#include "share/atm_process/atmosphere_process.hpp" + +// For MAM4 processes +#include + +// For MAM4 calcsize process (FIXME:should we include it in mam4 or mam_coupling??) +#include + +// For wetdep processes +#include + +// For component name +#include + +#include + +namespace scream { + +/* + * The class responsible to handle the aerosol wetscavenging + * + * The AD should store exactly ONE instance of this class stored + * in its list of subcomponents (the AD should make sure of this). + * + */ + +class MAMWetscav : public scream::AtmosphereProcess { + + using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; + + using const_view_2d = typename KT::template view_2d; + using const_view_1d = typename KT::template view_1d; // remove it if possible + + + // a thread team dispatched to a single vertical column + using ThreadTeam = mam4::ThreadTeam; + + public: + // Constructors + MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_wetscavenging"; } + + // Set the grid and input output variables + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + + + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol_in, const int nlev_in, + const mam_coupling::WetAtmosphere& wet_atm_in, + const mam_coupling::AerosolState& wet_aero_in, + const mam_coupling::DryAtmosphere& dry_atm_in, + const mam_coupling::AerosolState& dry_aero_in) { + ncol_pre_ = ncol_in; + nlev_pre_ = nlev_in; + wet_atm_pre_ = wet_atm_in; + wet_aero_pre_ = wet_aero_in; + dry_atm_pre_ = dry_atm_in; + dry_aero_pre_ = dry_aero_in; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const Kokkos::TeamPolicy::member_type& team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_vertical_layer_heights(team, dry_atm_pre_, i); + team.team_barrier(); // allows kernels below to use layer heights + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + team.team_barrier(); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + + }; // MAMWetscav::Preprocess + + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere& wet_atm, + const mam_coupling::AerosolState& wet_aero, + const mam_coupling::DryAtmosphere& dry_atm, + const mam_coupling::AerosolState& dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()(const Kokkos::TeamPolicy::member_type& team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, wet_aero_post_, i); + team.team_barrier(); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_post_, nlev_post_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // MAMWetscav::Postprocess + + + /* ----------------------------------------------- + * Local variables + * ------------------------------------------------ + */ + + // pre- and postprocessing scratch pads (for wet <-> dry conversions) + Preprocess preprocess_; + Postprocess postprocess_; + + + // Number of horizontal columns and vertical levels + int ncol_, nlev_; + + //Number of aerosol modes + static constexpr int ntot_amode_ = mam4::AeroConfig::num_modes(); + + //Extent for the e3sm's state vector for tracers + //--NOTE: The aerosol species are from index 16 to 40 ( or 15 to 39 in C++) + // but we define this variable from 0 to nvars_, where nvars_ is 39. + // Index 0 to 14 has no value + static constexpr int nvars_ = mam4::ndrop::nvars; + + + // atmospheric variables + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + view_2d cldn_prev_step_, cldt_prev_step_; // cloud fraction from the previous step, FIXME: they carry same info, we might remove one later + view_2d evapr_; + view_2d cldst_; + + view_2d rprdsh_; // rain production, shallow convection [kg/kg/s] + view_2d evapcsh_; + view_2d sh_frac_; + view_2d icwmrsh_; + + view_2d rprddp_; + view_2d evapcdp_; + view_2d dp_frac_; + view_2d icwmrdp_; + + + + // aerosol states + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // workspace manager for internal local variables + //ekat::WorkspaceManager workspace_mgr_; + mam_coupling::Buffer buffer_; + + mam4::WetDeposition wetdep_; + + + + std::shared_ptr m_grid; +}; // class MAMWetscav + +} // namespace scream + +#endif // EAMXX_MAM_WETSCAV_HPP diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt new file mode 100644 index 000000000000..1db8448fd845 --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt @@ -0,0 +1,30 @@ +include (ScreamUtils) + +# This test needs to be reworked for microphysics -- currently it's still using +# input for nucleation. + +# Create the test +CreateADUnitTest(mam4_wetscav_standalone + LABELS mam4_wetscav physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1) +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +set (RUN_T0 2021-10-12-45000) + +# ## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml new file mode 100644 index 000000000000..ce1c8d4eeefe --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml @@ -0,0 +1,68 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_wetscav] + mam_wetscav: + number_of_subcycles: ${NUM_SUBCYCLES} + compute_tendencies: [all] + lambda_low: 0.001 + +grids_manager: + Type: Mesh Free + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + cldn_prev_step: 0.5 + rprdsh: 1e-5 + rprddp: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + cldt_prev_step: 1e-5 + cldst: 1e-5 + qme: 1e-5 + prain: 1e-5 + evapr: 1e-5 + icwmrdp: 1e-5 + icwmrsh: 1e-5 + rprddp: 1e-5 + rprdsh: 1e-5 + sh_frac: 1e-5 + dp_frac: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + pbuf: 1e-5 # cloud-borne aerosols + dgncur_a: 1e-5 # aerosol particle diameter [m] + wetdens: 1e-5 # wet aerosol density [kg/m3] + qaerwat: 1e-5 # aerosol water [kg/kg] + dgnumwet: 1e-5 # wet aerosol diameter [m] + fracis: 1e-5 # fraction of transported species that are insoluble [fraction] + cldfrac_tot: 0.5 + pbl_height: 25.0 + phis : 0.1 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp b/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp new file mode 100644 index 000000000000..203b51681d05 --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp @@ -0,0 +1,69 @@ +#include + +#include "control/atmosphere_driver.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "physics/register_physics.hpp" +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/atm_process/atmosphere_process.hpp" + +#include "ekat/ekat_parse_yaml_file.hpp" +#include "ekat/logging/ekat_logger.hpp" + +#include + +namespace scream { + +TEST_CASE("mam4_wetscav-stand-alone", "") { + using namespace scream; + using namespace scream::control; + + // Create a comm + ekat::Comm atm_comm (MPI_COMM_WORLD); + + // Load ad parameter list + std::string fname = "input.yaml"; + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + logger.debug("yaml parsed."); + + // Time stepping parameters + const auto& ts = ad_params.sublist("time_stepping"); + const auto dt = ts.get("time_step"); + const auto nsteps = ts.get("number_of_steps"); + const auto t0_str = ts.get("run_t0"); + const auto t0 = util::str_to_time_stamp(t0_str); + + logger.info("running MAMWetscav standalone test with dt = {} for {} steps.", dt, nsteps); + EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); + + // Need to register products in the factory *before* we create any atm process or grids manager. + register_physics(); + register_mesh_free_grids_manager(); + register_diagnostics(); + logger.debug("products registered."); + + // Create the driver + AtmosphereDriver ad; + logger.debug("driver created."); + + // Init and run + ad.initialize(atm_comm,ad_params,t0); + logger.debug("driver initialized."); + + logger.info("Start time stepping loop ... [0%]"); + for (int i=0; i Date: Mon, 11 Mar 2024 09:31:06 -0600 Subject: [PATCH 066/477] wet_scav - update interface --- .../eamxx_mam_wetscav_process_interface.cpp | 7 +++++++ .../eamxx_mam_wetscav_process_interface.hpp | 21 +++++++++++++------ .../mam/impl/compute_particle_size.cpp | 9 ++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 components/eamxx/src/physics/mam/impl/compute_particle_size.cpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 7229419b378a..6aebcd75f89c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -1,4 +1,6 @@ +#include #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" +#include "scream_config.h" // for SCREAM_CIME_BUILD // NOTE: see the impl/ directory for the contents of the impl namespace #include "impl/compute_particle_size.cpp" @@ -27,6 +29,11 @@ MAMWetscav::MAMWetscav(const ekat::Comm &comm, */ } +AtmosphereProcessType MAMWetscav::type() const { + return AtmosphereProcessType::Physics; +} + +std::string MAMOptics::name() const { return "mam4_wet_scav"; } // ========================================================================================= void MAMWetscav::set_grids( const std::shared_ptr grids_manager) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index bec616e260a1..8a03529b5e1f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -20,6 +20,13 @@ #include #include +#ifndef KOKKOS_ENABLE_CUDA +#define protected_except_cuda public +#define private_except_cuda public +#else +#define protected_except_cuda protected +#define private_except_cuda private +#endif namespace scream { @@ -49,10 +56,10 @@ class MAMWetscav : public scream::AtmosphereProcess { MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The type of subcomponent - AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } - + AtmosphereProcessType + type() const override; // The name of the subcomponent - std::string name() const { return "mam_wetscavenging"; } + std::string name() const override; // Set the grid and input output variables void set_grids( @@ -95,12 +102,14 @@ class MAMWetscav : public scream::AtmosphereProcess { KOKKOS_INLINE_FUNCTION void operator()(const Kokkos::TeamPolicy::member_type& team) const { const int i = team.league_rank(); // column index - + // first, compute dry fields compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); + team.team_barrier(); + // second, we can use dry fields to compute dz, zmin, zint compute_vertical_layer_heights(team, dry_atm_pre_, i); - team.team_barrier(); // allows kernels below to use layer heights compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - team.team_barrier(); + team.team_barrier(); // allows kernels below to use layer heights } // operator() // number of horizontal columns and vertical levels diff --git a/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp b/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp new file mode 100644 index 000000000000..06a4c4cb3bea --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp @@ -0,0 +1,9 @@ +namespace scream::impl { + +/*KOKKOS_INLINE_FUNCTION +void compute_particle_size(int icol, int nlev_, //in + view_1d& state_q, view_1d& qqcw) { + +}*/ + +} // namespace scream::impl From 912e0bc1db38448d7d2b751de7c55ce082d3c247 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Mar 2024 11:54:44 -0600 Subject: [PATCH 067/477] wet_scav - using compute_calcsize_and_water_uptake_dr --- .../eamxx/src/physics/mam/CMakeLists.txt | 2 +- .../eamxx_mam_wetscav_process_interface.cpp | 91 +++++++++++++------ .../eamxx_mam_wetscav_process_interface.hpp | 17 ++-- .../eamxx/src/physics/register_physics.hpp | 2 + 4 files changed, 74 insertions(+), 38 deletions(-) diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index dd8cf6e6b304..3182e3837943 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -44,7 +44,7 @@ add_library(mam eamxx_mam_microphysics_process_interface.cpp eamxx_mam_optics_process_interface.cpp eamxx_mam_aci_process_interface.cpp - eamxx_mam_microphysics_process_interface.cpp) + eamxx_mam_wetscav_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 6aebcd75f89c..c778070f4487 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -33,7 +33,7 @@ AtmosphereProcessType MAMWetscav::type() const { return AtmosphereProcessType::Physics; } -std::string MAMOptics::name() const { return "mam4_wet_scav"; } +std::string MAMWetscav::name() const { return "mam4_wetscav"; } // ========================================================================================= void MAMWetscav::set_grids( const std::shared_ptr grids_manager) { @@ -371,7 +371,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { icwmrdp_ = get_field_out("icwmrdp") .get_view(); // ?? - + // set wet/dry aerosol state data (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { const char *int_nmr_field_name = @@ -418,7 +418,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - + printf("Working on wet_sav \n"); // preprocess input -- needs a scan for the calculation of all variables // needed by this process or setting up MAM4xx classes and their objects Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -450,6 +450,7 @@ void MAMWetscav::run_impl(const double dt) { */ // MUST: FIXME: Make sure state is not updated...only tendencies should be // updated!! +#if 0 static constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; int nspec_amode[ntot_amode_]; int lspectype_amode[maxd_aspectype][ntot_amode_]; @@ -490,8 +491,8 @@ void MAMWetscav::run_impl(const double dt) { // diagnostics for visible band summed over modes // Note: Need to compute inv density using indexing from e3sm - for(int imode = 0; imode < ntot_amode_; ++imode) { - const int nspec = nspec_amode[imode]; + // for(int imode = 0; imode < ntot_amode_; ++imode) { + // const int nspec = nspec_amode[imode]; for(int isp = 0; isp < nspec; ++isp) { const int idx = lspectype_amode[isp][imode] - 1; inv_density[imode][isp] = 1.0 / specdens_amode[idx]; @@ -503,18 +504,19 @@ void MAMWetscav::run_impl(const double dt) { nvars_); // MUST FIXME: make it is 3d view to avoid race condition view_2d qqcw("qqcw", nlev_, nvars_); - - //create a const column view of zeros + //create a const column view of zeros view_1d zeros_nlev("zeros_nlev", nlev_); Kokkos::deep_copy(zeros_nlev,0); const_view_1d const_zeros_nlev(zeros_nlev); +#endif + constexpr int pcnst = mam4::aero_model::pcnst; + constexpr int ntot_amode = mam4::AeroConfig::num_modes(); + constexpr int maxd_aspectype= mam4::ndrop::maxd_aspectype; mam4::AeroConfig aero_config; mam4::WetDeposition::Config wetdep_config;// = wetdep_.Config(); wetdep_.init(aero_config,wetdep_config);//FIXME: Should we call this in the initialize???? - - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -523,6 +525,51 @@ void MAMWetscav::run_impl(const double dt) { policy, KOKKOS_LAMBDA(const ThreadTeam &team) { const int icol = team.league_rank(); // column index*/ + // call wetdep for computing....add mod=re descriptive comment + // here? + auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); + // set surface state data + const haero::Surface + sfc{}; // sfc object is NEVER used in wetdep process + + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = + mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); + + // compute calcsize and + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0, nlev_), [&](int kk) { + + Real state_q[pcnst] = {}; + Real qqcw[pcnst] = {}; + + int nspec_amode[ntot_amode]; + int lspectype_amode[maxd_aspectype][ntot_amode]; + int lmassptr_amode[maxd_aspectype][ntot_amode]; + Real specdens_amode[maxd_aspectype]; + Real spechygro[maxd_aspectype]; + + mam4::utils::extract_stateq_from_prognostics(progs, atm, state_q, kk); + mam4::utils::extract_qqcw_from_prognostics(progs, qqcw, kk); + team.team_barrier(); + + Real mean_std_dev_nmodes[ntot_amode] = {}; + Real dgnumwet_m_kk[ntot_amode] = {}; + Real qaerwat_m_kk[ntot_amode] = {}; + + mam4::modal_aer_opt::compute_calcsize_and_water_uptake_dr( + dry_atm_.p_mid(icol, kk), dry_atm_.T_mid(icol, kk), + cldn_prev_step_(icol, kk), state_q, qqcw, dt, // in + nspec_amode, lspectype_amode, specdens_amode, lmassptr_amode, spechygro, + mean_std_dev_nmodes, dgnumwet_m_kk, qaerwat_m_kk); + + team.team_barrier(); + mam4::utils::inject_qqcw_to_prognostics(qqcw, progs, kk); + mam4::utils::inject_stateq_to_prognostics(state_q, progs, kk); + + }); // klev parallel_for loop + +#if 0 Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0, nlev_), [&](int klev) { view_1d state_q_k = ekat::subview( @@ -585,19 +632,7 @@ void MAMWetscav::run_impl(const double dt) { cldn_prev_step_(icol, klev), dgnumdry_m_kk, dgnumwet_m_kk, qaerwat_m_kk); }); // klev parallel_for loop - - // call wetdep for computing....add mod=re descriptive comment - // here? - - auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); - - // set surface state data - const haero::Surface - sfc{}; // sfc object is NEVER used in wetdep process - - // fetch column-specific subviews into aerosol prognostics - mam4::Prognostics progs = - mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); +#endif // set up diagnostics mam4::Diagnostics diags(nlev_); @@ -606,18 +641,16 @@ void MAMWetscav::run_impl(const double dt) { diags.shallow_convective_precipitation_evaporation = ekat::subview(evapcsh_, icol); - diags.deep_convective_cloud_fraction = ekat::subview(dp_frac_, - icol); + icol); //std::cout<<"BALLI:"< grids_manager) override; - + // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; void init_buffers(const ATMBufferManager &buffer_manager) override; @@ -111,7 +111,7 @@ class MAMWetscav : public scream::AtmosphereProcess { compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); team.team_barrier(); // allows kernels below to use layer heights } // operator() - + // number of horizontal columns and vertical levels int ncol_pre_, nlev_pre_; @@ -166,7 +166,7 @@ class MAMWetscav : public scream::AtmosphereProcess { Preprocess preprocess_; Postprocess postprocess_; - + // Number of horizontal columns and vertical levels int ncol_, nlev_; @@ -177,7 +177,8 @@ class MAMWetscav : public scream::AtmosphereProcess { //--NOTE: The aerosol species are from index 16 to 40 ( or 15 to 39 in C++) // but we define this variable from 0 to nvars_, where nvars_ is 39. // Index 0 to 14 has no value - static constexpr int nvars_ = mam4::ndrop::nvars; + // FIXME: do we need nvars_? + static constexpr int nvars_ = mam4::aero_model::pcnst; // atmospheric variables @@ -197,9 +198,9 @@ class MAMWetscav : public scream::AtmosphereProcess { view_2d evapcdp_; view_2d dp_frac_; view_2d icwmrdp_; - - + + // aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_; @@ -209,8 +210,8 @@ class MAMWetscav : public scream::AtmosphereProcess { mam4::WetDeposition wetdep_; - - + + std::shared_ptr m_grid; }; // class MAMWetscav diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 36afabf8de97..44becc2f3e6a 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -27,6 +27,7 @@ #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" #include "physics/mam/eamxx_mam_aci_process_interface.hpp" +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -64,6 +65,7 @@ inline void register_physics () { proc_factory.register_product("mam4_micro",&create_atmosphere_process); proc_factory.register_product("mam4_optics",&create_atmosphere_process); proc_factory.register_product("mam4_aci",&create_atmosphere_process); + proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 68c9a9dc45ce2a3174bc94dc1b925021a1bab7f4 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 11 Mar 2024 11:57:33 -0600 Subject: [PATCH 068/477] wet_scav - delete old code. --- .../eamxx_mam_wetscav_process_interface.cpp | 128 +----------------- 1 file changed, 3 insertions(+), 125 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c778070f4487..d198a170646c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -418,6 +418,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + printf("Working on wet_sav \n"); // preprocess input -- needs a scan for the calculation of all variables // needed by this process or setting up MAM4xx classes and their objects @@ -450,65 +451,6 @@ void MAMWetscav::run_impl(const double dt) { */ // MUST: FIXME: Make sure state is not updated...only tendencies should be // updated!! -#if 0 - static constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; - int nspec_amode[ntot_amode_]; - int lspectype_amode[maxd_aspectype][ntot_amode_]; - mam4::Real specdens_amode[maxd_aspectype]; - int lmassptr_amode[maxd_aspectype][ntot_amode_]; - Real spechygro[maxd_aspectype]; - - int numptr_amode[ntot_amode_]; - int mam_idx[ntot_amode_][mam4::ndrop::nspec_max]; - int mam_cnst_idx[ntot_amode_][mam4::ndrop::nspec_max]; - - mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, - numptr_amode, specdens_amode, spechygro, - mam_idx, mam_cnst_idx); - - Real inv_density[ntot_amode_][mam4::AeroConfig::num_aerosol_ids()] = {}; - Real num2vol_ratio_min[ntot_amode_] = {}; - Real num2vol_ratio_max[ntot_amode_] = {}; - Real num2vol_ratio_max_nmodes[ntot_amode_] = {}; - Real num2vol_ratio_min_nmodes[ntot_amode_] = {}; - Real num2vol_ratio_nom_nmodes[ntot_amode_] = {}; - Real dgnmin_nmodes[ntot_amode_] = {}; - Real dgnmax_nmodes[ntot_amode_] = {}; - Real dgnnom_nmodes[ntot_amode_] = {}; - Real mean_std_dev_nmodes[ntot_amode_] = {}; - - bool noxf_acc2ait[mam4::AeroConfig::num_aerosol_ids()] = {}; - int n_common_species_ait_accum = {}; - int ait_spec_in_acc[mam4::AeroConfig::num_aerosol_ids()] = {}; - int acc_spec_in_ait[mam4::AeroConfig::num_aerosol_ids()] = {}; - mam4::modal_aero_calcsize::init_calcsize( - inv_density, num2vol_ratio_min, num2vol_ratio_max, - num2vol_ratio_max_nmodes, num2vol_ratio_min_nmodes, - num2vol_ratio_nom_nmodes, dgnmin_nmodes, dgnmax_nmodes, dgnnom_nmodes, - mean_std_dev_nmodes, noxf_acc2ait, n_common_species_ait_accum, - ait_spec_in_acc, acc_spec_in_ait); - - // diagnostics for visible band summed over modes - - // Note: Need to compute inv density using indexing from e3sm - // for(int imode = 0; imode < ntot_amode_; ++imode) { - // const int nspec = nspec_amode[imode]; - for(int isp = 0; isp < nspec; ++isp) { - const int idx = lspectype_amode[isp][imode] - 1; - inv_density[imode][isp] = 1.0 / specdens_amode[idx]; - } // isp - } // imode - - view_2d state_q( - "state_q", nlev_, - nvars_); // MUST FIXME: make it is 3d view to avoid race condition - view_2d qqcw("qqcw", nlev_, nvars_); - - //create a const column view of zeros - view_1d zeros_nlev("zeros_nlev", nlev_); - Kokkos::deep_copy(zeros_nlev,0); - const_view_1d const_zeros_nlev(zeros_nlev); -#endif constexpr int pcnst = mam4::aero_model::pcnst; constexpr int ntot_amode = mam4::AeroConfig::num_modes(); constexpr int maxd_aspectype= mam4::ndrop::maxd_aspectype; @@ -563,77 +505,13 @@ void MAMWetscav::run_impl(const double dt) { nspec_amode, lspectype_amode, specdens_amode, lmassptr_amode, spechygro, mean_std_dev_nmodes, dgnumwet_m_kk, qaerwat_m_kk); - team.team_barrier(); + mam4::utils::inject_qqcw_to_prognostics(qqcw, progs, kk); mam4::utils::inject_stateq_to_prognostics(state_q, progs, kk); }); // klev parallel_for loop -#if 0 - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0, nlev_), [&](int klev) { - view_1d state_q_k = ekat::subview( - state_q, klev); // 1d view of size (nvars_) for storing all - // gasses and aerosols - mam_coupling::state_q_for_column_at_one_lev(dry_aero_, icol, klev, - state_q_k); - - view_1d qqcw_k = - ekat::subview(qqcw, klev); // 1d view of size (nvars_) for - // storing all gasses and aerosols - mam_coupling::qqcw_for_column_at_one_lev(dry_aero_, icol, klev, - qqcw_k); - - static constexpr bool do_adjust = true, do_aitacc_transfer = true, - update_mmr = true; - - Real dgncur_c_kk[ntot_amode_] = {}; - Real dgnumdry_m_kk[ntot_amode_] = {}; - // Calculate aerosol size distribution parameters and aerosol - // water - // uptake - mam4::modal_aero_calcsize::modal_aero_calcsize_sub( - state_q_k.data(), // in - qqcw_k.data(), // in/out - dt, do_adjust, do_aitacc_transfer, update_mmr, lmassptr_amode, - numptr_amode, - inv_density, // in - num2vol_ratio_min, num2vol_ratio_max, - num2vol_ratio_max_nmodes, num2vol_ratio_min_nmodes, - num2vol_ratio_nom_nmodes, dgnmin_nmodes, dgnmax_nmodes, - dgnnom_nmodes, mean_std_dev_nmodes, - // outputs - noxf_acc2ait, n_common_species_ait_accum, ait_spec_in_acc, - acc_spec_in_ait, dgnumdry_m_kk, dgncur_c_kk); - - Real dgnumwet_m_kk[ntot_amode_] = {}; - Real qaerwat_m_kk[ntot_amode_] = {}; - // std::cout< Date: Mon, 11 Mar 2024 14:34:53 -0600 Subject: [PATCH 069/477] wet_scav - remove private member of parallel_for --- .../eamxx_mam_wetscav_process_interface.cpp | 56 +++++++++++++------ .../uncoupled/mam4_wet_scav/CMakeLists.txt | 2 +- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index d198a170646c..cb4ef5130932 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -412,6 +412,9 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // objects in the argument list preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // ========================================================================================= @@ -459,6 +462,23 @@ void MAMWetscav::run_impl(const double dt) { mam4::WetDeposition::Config wetdep_config;// = wetdep_.Config(); wetdep_.init(aero_config,wetdep_config);//FIXME: Should we call this in the initialize???? + // NOTE! we need a const mam_coupling::DryAtmosphere dry_atm for gpu access. + // We cannot use member of this class inside of the parallel_for + const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + const auto &dry_aero = dry_aero_; + const auto &cldn_prev_step =cldn_prev_step_; + const auto &rprdsh = rprdsh_; + const auto &evapcsh = evapcsh_; + const auto &dp_frac=dp_frac_; + const auto &sh_frac=sh_frac_; + const auto &icwmrsh=icwmrsh_; + const auto &cldt_prev_step=cldt_prev_step_; + const auto &cldst=cldst_; + const auto &evapr=evapr_; + const auto &rprddp=rprddp_; + const auto &evapcdp=evapcdp_; + const auto &icwmrdp=icwmrdp_; + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -469,14 +489,14 @@ void MAMWetscav::run_impl(const double dt) { // call wetdep for computing....add mod=re descriptive comment // here? - auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); + auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); // set surface state data const haero::Surface sfc{}; // sfc object is NEVER used in wetdep process // fetch column-specific subviews into aerosol prognostics mam4::Prognostics progs = - mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); + mam_coupling::aerosols_for_column(dry_aero, icol); // compute calcsize and Kokkos::parallel_for( @@ -492,6 +512,7 @@ void MAMWetscav::run_impl(const double dt) { Real spechygro[maxd_aspectype]; mam4::utils::extract_stateq_from_prognostics(progs, atm, state_q, kk); + mam4::utils::extract_qqcw_from_prognostics(progs, qqcw, kk); team.team_barrier(); @@ -500,47 +521,48 @@ void MAMWetscav::run_impl(const double dt) { Real qaerwat_m_kk[ntot_amode] = {}; mam4::modal_aer_opt::compute_calcsize_and_water_uptake_dr( - dry_atm_.p_mid(icol, kk), dry_atm_.T_mid(icol, kk), - cldn_prev_step_(icol, kk), state_q, qqcw, dt, // in + dry_atm.p_mid(icol, kk), dry_atm.T_mid(icol, kk), + cldn_prev_step(icol, kk), state_q, qqcw, dt, // in nspec_amode, lspectype_amode, specdens_amode, lmassptr_amode, spechygro, mean_std_dev_nmodes, dgnumwet_m_kk, qaerwat_m_kk); - mam4::utils::inject_qqcw_to_prognostics(qqcw, progs, kk); mam4::utils::inject_stateq_to_prognostics(state_q, progs, kk); - }); // klev parallel_for loop team.team_barrier(); // set up diagnostics mam4::Diagnostics diags(nlev_); diags.shallow_convective_precipitation_production = - ekat::subview(rprdsh_, icol); + ekat::subview(rprdsh, icol); diags.shallow_convective_precipitation_evaporation = - ekat::subview(evapcsh_, icol); + ekat::subview(evapcsh, icol); + - diags.deep_convective_cloud_fraction = ekat::subview(dp_frac_, + diags.deep_convective_cloud_fraction = ekat::subview(dp_frac, icol); //std::cout<<"BALLI:"< Date: Mon, 11 Mar 2024 17:56:18 -0600 Subject: [PATCH 070/477] wet_scav - add dig variables. Unit tests is working. --- .../eamxx_mam_wetscav_process_interface.cpp | 111 ++++++++++++------ .../eamxx_mam_wetscav_process_interface.hpp | 10 +- 2 files changed, 84 insertions(+), 37 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index cb4ef5130932..09a808b79b13 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -340,38 +340,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } - // Other required variables - cldn_prev_step_ = get_field_out("cldn_prev_step").get_view< Real **>(); - cldt_prev_step_ = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - cldst_ = get_field_out("cldst").get_view< Real **>();//?? - evapr_ = get_field_out("evapr").get_view< Real **>(); - rprdsh_ = - get_field_out("rprdsh").get_view(); // rain production, shallow - // convection [kg/kg/s] - evapcsh_ = - get_field_out("evapcsh") - .get_view(); // Evaporation rate of shallow convective - // precipitation >=0. [kg/kg/s] - - sh_frac_ = get_field_out("sh_frac") - .get_view(); // Shallow convective cloud fraction [fraction] - - rprddp_ = - get_field_out("rprddp") - .get_view(); // rain production, deep convection [kg/kg/s] - - evapcdp_ = get_field_out("evapcdp") - .get_view(); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] - dp_frac_ = get_field_out("dp_frac") - .get_view(); // Deep convective cloud fraction [fraction] - - icwmrsh_ = get_field_out("icwmrsh") - .get_view(); // ?? - - icwmrdp_ = get_field_out("icwmrdp") - .get_view(); // ?? - // set wet/dry aerosol state data (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { const char *int_nmr_field_name = @@ -407,6 +375,39 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } + // Other required variables + cldn_prev_step_ = get_field_out("cldn_prev_step").get_view< Real **>(); + cldt_prev_step_ = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + cldst_ = get_field_out("cldst").get_view< Real **>();//?? + evapr_ = get_field_out("evapr").get_view< Real **>(); + rprdsh_ = + get_field_out("rprdsh").get_view(); // rain production, shallow + // convection [kg/kg/s] + evapcsh_ = + get_field_out("evapcsh") + .get_view(); // Evaporation rate of shallow convective + // precipitation >=0. [kg/kg/s] + + sh_frac_ = get_field_out("sh_frac") + .get_view(); // Shallow convective cloud fraction [fraction] + + rprddp_ = + get_field_out("rprddp") + .get_view(); // rain production, deep convection [kg/kg/s] + + evapcdp_ = get_field_out("evapcdp") + .get_view(); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + dp_frac_ = get_field_out("dp_frac") + .get_view(); // Deep convective cloud fraction [fraction] + + icwmrsh_ = get_field_out("icwmrsh") + .get_view(); // ?? + + icwmrdp_ = get_field_out("icwmrdp") + .get_view(); // ?? + + // set up our preprocess/postprocess functors // Here we initialize (not compute) objects in preprocess struct using the // objects in the argument list @@ -415,6 +416,24 @@ void MAMWetscav::initialize_impl(const RunType run_type) { postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + // wetdep + constexpr int pcnst = mam4::aero_model::pcnst; + + tracer_mixing_ratio_work_ = view_3d("tracer_mixing_ratio_work_", ncol_, nlev_, pcnst); + d_tracer_mixing_ratio_dt_work_ = view_3d("d_tracer_mixing_ratio_dt_work_", ncol_, nlev_, pcnst); + + aerosol_wet_deposition_interstitial_work_ = view_2d("aerosol_wet_deposition_interstitial",ncol_, nlev_); + aerosol_wet_deposition_cloud_water_work_ = view_2d("aerosol_wet_deposition_cloud_water",ncol_, nlev_); + + for (int m = 0; m < mam_coupling::num_aero_modes(); m++) + { + wet_geometric_mean_diameter_i_work_[m] + = view_2d("wet_geometric_mean_diameter_i_work_1",ncol_, nlev_); + } + + total_convective_detrainment_work_= view_2d("total_convective_detrainment_work_",ncol_, nlev_); + } // ========================================================================================= @@ -455,7 +474,7 @@ void MAMWetscav::run_impl(const double dt) { // MUST: FIXME: Make sure state is not updated...only tendencies should be // updated!! constexpr int pcnst = mam4::aero_model::pcnst; - constexpr int ntot_amode = mam4::AeroConfig::num_modes(); + constexpr int ntot_amode = mam_coupling::num_aero_modes(); constexpr int maxd_aspectype= mam4::ndrop::maxd_aspectype; mam4::AeroConfig aero_config; @@ -478,7 +497,16 @@ void MAMWetscav::run_impl(const double dt) { const auto &rprddp=rprddp_; const auto &evapcdp=evapcdp_; const auto &icwmrdp=icwmrdp_; - + const auto &tracer_mixing_ratio_work=tracer_mixing_ratio_work_; + const auto &d_tracer_mixing_ratio_dt_work=d_tracer_mixing_ratio_dt_work_; + const auto &aerosol_wet_deposition_interstitial_work=aerosol_wet_deposition_interstitial_work_; + const auto &aerosol_wet_deposition_cloud_water_work=aerosol_wet_deposition_cloud_water_work_; + const auto &total_convective_detrainment_work=total_convective_detrainment_work_; + view_2d wet_geometric_mean_diameter_i_work[ntot_amode]; + for (int m = 0; m < ntot_amode; m++) + { + wet_geometric_mean_diameter_i_work[m] = wet_geometric_mean_diameter_i_work_[m]; + } const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -546,6 +574,7 @@ void MAMWetscav::run_impl(const double dt) { diags.shallow_convective_cloud_condensate = ekat::subview(icwmrsh_, icol); // FIXME: why are we setting deep_convective_cloud_fraction to zeros_nlev? // diags.deep_convective_cloud_fraction = zeros_nlev; + // FXIME: shallow_convective_cloud_fraction was previously set to dp_frac_icol. diags.shallow_convective_cloud_fraction = ekat::subview(cldt_prev_step, icol); diags.stratiform_cloud_fraction = ekat::subview(cldst, icol); diags.evaporation_of_falling_precipitation = ekat::subview(evapr, icol); @@ -556,7 +585,19 @@ void MAMWetscav::run_impl(const double dt) { ekat::subview(evapcdp, icol); diags.deep_convective_cloud_condensate = ekat::subview(icwmrdp, icol); - // setup tendencies + diags.tracer_mixing_ratio = ekat::subview(tracer_mixing_ratio_work, icol); + diags.d_tracer_mixing_ratio_dt = ekat::subview(d_tracer_mixing_ratio_dt_work, icol); + + diags.aerosol_wet_deposition_interstitial = ekat::subview(aerosol_wet_deposition_interstitial_work, icol); + diags.aerosol_wet_deposition_cloud_water = ekat::subview(aerosol_wet_deposition_cloud_water_work, icol); + + // FIXME where is dp computed? calcsize? + for (int m = 0; m < ntot_amode; m++) + { + diags.wet_geometric_mean_diameter_i[m] = ekat::subview(wet_geometric_mean_diameter_i_work[m], icol); + } + // // setup tendencies + diags.total_convective_detrainment = ekat::subview(total_convective_detrainment_work, icol); mam4::Tendencies tends{}; #if 1 wetdep_.compute_tendencies(aero_config, team, 0, dt, atm, sfc, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index b6a1dc67a1ca..e85a6c3f3c61 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -43,6 +43,7 @@ class MAMWetscav : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; using view_1d = typename KT::template view_1d; using view_2d = typename KT::template view_2d; + using view_3d = typename KT::template view_3d; using const_view_2d = typename KT::template view_2d; using const_view_1d = typename KT::template view_1d; // remove it if possible @@ -119,7 +120,6 @@ class MAMWetscav : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMWetscav::Preprocess // Postprocessing functor @@ -199,7 +199,13 @@ class MAMWetscav : public scream::AtmosphereProcess { view_2d dp_frac_; view_2d icwmrdp_; - + // wet dep + view_3d tracer_mixing_ratio_work_; + view_3d d_tracer_mixing_ratio_dt_work_; + view_2d aerosol_wet_deposition_interstitial_work_; + view_2d aerosol_wet_deposition_cloud_water_work_; + view_2d wet_geometric_mean_diameter_i_work_[mam4::AeroConfig::num_modes()]; + view_2d total_convective_detrainment_work_; // aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_; From 7d3cbff26cbe52e8c1a3f96c507f3443d31961cd Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Fri, 22 Mar 2024 14:10:37 -0600 Subject: [PATCH 071/477] wet_scav - update interface --- .../eamxx_mam_wetscav_process_interface.cpp | 260 +++++++----------- .../eamxx_mam_wetscav_process_interface.hpp | 39 +-- 2 files changed, 102 insertions(+), 197 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 09a808b79b13..48643fe084a3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -259,6 +259,19 @@ void MAMWetscav::set_grids( grid_name); // cloud ice number conc add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); // vertical pressure velocity + + // FIXME: units + add_field("dlf", scalar3d_layout_mid, n_unit, + grid_name); // + add_field("dp_ccf", scalar3d_layout_mid, n_unit, + grid_name); // + add_field("sh_ccf", scalar3d_layout_mid, n_unit, + grid_name); // + // + add_field("aerdepwetis", scalar3d_layout_mid, n_unit, + grid_name); // + add_field("aerdepwetcw", scalar3d_layout_mid, n_unit, + grid_name); // } // ========================================================================================= @@ -374,40 +387,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } } - - // Other required variables - cldn_prev_step_ = get_field_out("cldn_prev_step").get_view< Real **>(); - cldt_prev_step_ = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - cldst_ = get_field_out("cldst").get_view< Real **>();//?? - evapr_ = get_field_out("evapr").get_view< Real **>(); - rprdsh_ = - get_field_out("rprdsh").get_view(); // rain production, shallow - // convection [kg/kg/s] - evapcsh_ = - get_field_out("evapcsh") - .get_view(); // Evaporation rate of shallow convective - // precipitation >=0. [kg/kg/s] - - sh_frac_ = get_field_out("sh_frac") - .get_view(); // Shallow convective cloud fraction [fraction] - - rprddp_ = - get_field_out("rprddp") - .get_view(); // rain production, deep convection [kg/kg/s] - - evapcdp_ = get_field_out("evapcdp") - .get_view(); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] - dp_frac_ = get_field_out("dp_frac") - .get_view(); // Deep convective cloud fraction [fraction] - - icwmrsh_ = get_field_out("icwmrsh") - .get_view(); // ?? - - icwmrdp_ = get_field_out("icwmrdp") - .get_view(); // ?? - - // set up our preprocess/postprocess functors // Here we initialize (not compute) objects in preprocess struct using the // objects in the argument list @@ -419,21 +398,11 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // wetdep constexpr int pcnst = mam4::aero_model::pcnst; - - tracer_mixing_ratio_work_ = view_3d("tracer_mixing_ratio_work_", ncol_, nlev_, pcnst); - d_tracer_mixing_ratio_dt_work_ = view_3d("d_tracer_mixing_ratio_dt_work_", ncol_, nlev_, pcnst); - - aerosol_wet_deposition_interstitial_work_ = view_2d("aerosol_wet_deposition_interstitial",ncol_, nlev_); - aerosol_wet_deposition_cloud_water_work_ = view_2d("aerosol_wet_deposition_cloud_water",ncol_, nlev_); - - for (int m = 0; m < mam_coupling::num_aero_modes(); m++) - { - wet_geometric_mean_diameter_i_work_[m] - = view_2d("wet_geometric_mean_diameter_i_work_1",ncol_, nlev_); - } - - total_convective_detrainment_work_= view_2d("total_convective_detrainment_work_",ncol_, nlev_); - + // FIXME: qqcw_sav_ should be part of work_ + Kokkos::View + qqcw_sav_("qqcw_sav", mam4::nlev); + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); + work_ = view_2d("work", ncol_, work_len); } // ========================================================================================= @@ -473,40 +442,43 @@ void MAMWetscav::run_impl(const double dt) { */ // MUST: FIXME: Make sure state is not updated...only tendencies should be // updated!! - constexpr int pcnst = mam4::aero_model::pcnst; - constexpr int ntot_amode = mam_coupling::num_aero_modes(); - constexpr int maxd_aspectype= mam4::ndrop::maxd_aspectype; - - mam4::AeroConfig aero_config; - mam4::WetDeposition::Config wetdep_config;// = wetdep_.Config(); - wetdep_.init(aero_config,wetdep_config);//FIXME: Should we call this in the initialize???? - // NOTE! we need a const mam_coupling::DryAtmosphere dry_atm for gpu access. // We cannot use member of this class inside of the parallel_for const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; - const auto &cldn_prev_step =cldn_prev_step_; - const auto &rprdsh = rprdsh_; - const auto &evapcsh = evapcsh_; - const auto &dp_frac=dp_frac_; - const auto &sh_frac=sh_frac_; - const auto &icwmrsh=icwmrsh_; - const auto &cldt_prev_step=cldt_prev_step_; - const auto &cldst=cldst_; - const auto &evapr=evapr_; - const auto &rprddp=rprddp_; - const auto &evapcdp=evapcdp_; - const auto &icwmrdp=icwmrdp_; - const auto &tracer_mixing_ratio_work=tracer_mixing_ratio_work_; - const auto &d_tracer_mixing_ratio_dt_work=d_tracer_mixing_ratio_dt_work_; - const auto &aerosol_wet_deposition_interstitial_work=aerosol_wet_deposition_interstitial_work_; - const auto &aerosol_wet_deposition_cloud_water_work=aerosol_wet_deposition_cloud_water_work_; - const auto &total_convective_detrainment_work=total_convective_detrainment_work_; - view_2d wet_geometric_mean_diameter_i_work[ntot_amode]; - for (int m = 0; m < ntot_amode; m++) - { - wet_geometric_mean_diameter_i_work[m] = wet_geometric_mean_diameter_i_work_[m]; - } + const auto &qqcw_sav = qqcw_sav_; + const auto &work = work_; + + // inputs + auto dlf = get_field_in("dlf").get_view(); + auto dp_ccf = get_field_in("dp_ccf").get_view(); + auto sh_ccf = get_field_in("sh_ccf").get_view(); + auto cldn_prev_step = get_field_out("cldn_prev_step").get_view< Real **>(); + // where is cldt_prev_step used? + auto cldt_prev_step = get_field_in("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + auto cldst = get_field_in("cldst").get_view< Real **>();//?? + auto evapr = get_field_in("evapr").get_view< Real **>(); + auto rprdsh = get_field_in("rprdsh").get_view(); // rain production, shallow + // convection [kg/kg/s] + auto evapcsh = get_field_in("evapcsh").get_view(); // Evaporation rate of shallow convective + // precipitation >=0. [kg/kg/s] + auto sh_frac = get_field_in("sh_frac").get_view(); // Shallow convective cloud fraction [fraction] + auto rprddp = get_field_in("rprddp").get_view(); // rain production, deep convection [kg/kg/s] + auto evapcdp = get_field_in("evapcdp").get_view(); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + auto dp_frac = get_field_in("dp_frac") + .get_view(); // Deep convective cloud fraction [fraction] + + auto icwmrsh = get_field_in("icwmrsh") + .get_view(); // ?? + + auto icwmrdp = get_field_in("icwmrdp") + .get_view(); // ?? + + // outputs + const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); + const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -515,95 +487,61 @@ void MAMWetscav::run_impl(const double dt) { policy, KOKKOS_LAMBDA(const ThreadTeam &team) { const int icol = team.league_rank(); // column index*/ - // call wetdep for computing....add mod=re descriptive comment - // here? auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); // set surface state data - const haero::Surface - sfc{}; // sfc object is NEVER used in wetdep process - // fetch column-specific subviews into aerosol prognostics mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aero, icol); - - // compute calcsize and - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0, nlev_), [&](int kk) { - - Real state_q[pcnst] = {}; - Real qqcw[pcnst] = {}; - - int nspec_amode[ntot_amode]; - int lspectype_amode[maxd_aspectype][ntot_amode]; - int lmassptr_amode[maxd_aspectype][ntot_amode]; - Real specdens_amode[maxd_aspectype]; - Real spechygro[maxd_aspectype]; - - mam4::utils::extract_stateq_from_prognostics(progs, atm, state_q, kk); - - mam4::utils::extract_qqcw_from_prognostics(progs, qqcw, kk); - team.team_barrier(); - - Real mean_std_dev_nmodes[ntot_amode] = {}; - Real dgnumwet_m_kk[ntot_amode] = {}; - Real qaerwat_m_kk[ntot_amode] = {}; - - mam4::modal_aer_opt::compute_calcsize_and_water_uptake_dr( - dry_atm.p_mid(icol, kk), dry_atm.T_mid(icol, kk), - cldn_prev_step(icol, kk), state_q, qqcw, dt, // in - nspec_amode, lspectype_amode, specdens_amode, lmassptr_amode, spechygro, - mean_std_dev_nmodes, dgnumwet_m_kk, qaerwat_m_kk); - - mam4::utils::inject_qqcw_to_prognostics(qqcw, progs, kk); - mam4::utils::inject_stateq_to_prognostics(state_q, progs, kk); - }); // klev parallel_for loop - - team.team_barrier(); - // set up diagnostics - mam4::Diagnostics diags(nlev_); - diags.shallow_convective_precipitation_production = - ekat::subview(rprdsh, icol); - diags.shallow_convective_precipitation_evaporation = - ekat::subview(evapcsh, icol); - - - diags.deep_convective_cloud_fraction = ekat::subview(dp_frac, - icol); - //std::cout<<"BALLI:"< + qqcw_sav_; // aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_; - // workspace manager for internal local variables - //ekat::WorkspaceManager workspace_mgr_; mam_coupling::Buffer buffer_; - mam4::WetDeposition wetdep_; - - - std::shared_ptr m_grid; }; // class MAMWetscav From 8363b750b59881d56c981b8d6ac9297648c94050 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 26 Mar 2024 16:32:30 -0600 Subject: [PATCH 072/477] wet_scav - test is working in cpu/gpus. --- .../eamxx_mam_wetscav_process_interface.cpp | 51 +++++++++++-------- .../tests/uncoupled/mam4_wet_scav/input.yaml | 4 +- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 48643fe084a3..60f352f06e34 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -261,11 +261,11 @@ void MAMWetscav::set_grids( grid_name); // vertical pressure velocity // FIXME: units - add_field("dlf", scalar3d_layout_mid, n_unit, + add_field("dlf", scalar3d_layout_mid, n_unit, grid_name); // - add_field("dp_ccf", scalar3d_layout_mid, n_unit, + add_field("dp_ccf", scalar3d_layout_mid, n_unit, grid_name); // - add_field("sh_ccf", scalar3d_layout_mid, n_unit, + add_field("sh_ccf", scalar3d_layout_mid, n_unit, grid_name); // // add_field("aerdepwetis", scalar3d_layout_mid, n_unit, @@ -399,8 +399,10 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // wetdep constexpr int pcnst = mam4::aero_model::pcnst; // FIXME: qqcw_sav_ should be part of work_ - Kokkos::View - qqcw_sav_("qqcw_sav", mam4::nlev); + // we need to add columns + qqcw_sav_ =Kokkos::View("qqcw_sav", mam4::nlev); + + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); } @@ -449,30 +451,33 @@ void MAMWetscav::run_impl(const double dt) { const auto &qqcw_sav = qqcw_sav_; const auto &work = work_; - // inputs - auto dlf = get_field_in("dlf").get_view(); - auto dp_ccf = get_field_in("dp_ccf").get_view(); - auto sh_ccf = get_field_in("sh_ccf").get_view(); + // inputs/outputs + auto dlf = get_field_out("dlf").get_view(); + auto dp_ccf = get_field_out("dp_ccf").get_view(); + auto sh_ccf = get_field_out("sh_ccf").get_view(); + auto cldn_prev_step = get_field_out("cldn_prev_step").get_view< Real **>(); + // where is cldt_prev_step used? - auto cldt_prev_step = get_field_in("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - auto cldst = get_field_in("cldst").get_view< Real **>();//?? - auto evapr = get_field_in("evapr").get_view< Real **>(); - auto rprdsh = get_field_in("rprdsh").get_view(); // rain production, shallow - // convection [kg/kg/s] - auto evapcsh = get_field_in("evapcsh").get_view(); // Evaporation rate of shallow convective + auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + auto cldst = get_field_out("cldst").get_view< Real **>();//?? + auto evapr = get_field_out("evapr").get_view< Real **>(); + auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow + + // convection [kg/kg/s] + auto evapcsh = get_field_out("evapcsh").get_view(); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] - auto sh_frac = get_field_in("sh_frac").get_view(); // Shallow convective cloud fraction [fraction] - auto rprddp = get_field_in("rprddp").get_view(); // rain production, deep convection [kg/kg/s] - auto evapcdp = get_field_in("evapcdp").get_view(); // Evaporation rate of deep convective + auto sh_frac = get_field_out("sh_frac").get_view(); // Shallow convective cloud fraction [fraction] + auto rprddp = get_field_out("rprddp").get_view(); // rain production, deep convection [kg/kg/s] + auto evapcdp = get_field_out("evapcdp").get_view(); // Evaporation rate of deep convective // precipitation >=0. [kg/kg/s] - auto dp_frac = get_field_in("dp_frac") + auto dp_frac = get_field_out("dp_frac") .get_view(); // Deep convective cloud fraction [fraction] - auto icwmrsh = get_field_in("icwmrsh") + auto icwmrsh = get_field_out("icwmrsh") .get_view(); // ?? - auto icwmrdp = get_field_in("icwmrdp") + auto icwmrdp = get_field_out("icwmrdp") .get_view(); // ?? // outputs @@ -531,7 +536,8 @@ void MAMWetscav::run_impl(const double dt) { // diags.aerosol_wet_deposition_cloud_water; auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol);\ auto work_icol = ekat::subview(work, icol); - + // auto qqcw_sav_icol = ekat::subview(qqcw_sav,icol); +#if 1 mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, // inputs cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, @@ -542,6 +548,7 @@ void MAMWetscav::run_impl(const double dt) { aerdepwetis_icol, aerdepwetcw_icol, // FIXME remove qqcw_sav qqcw_sav, work_icol); +#endif }); // icol parallel_for loop /* diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml index ce1c8d4eeefe..c73e2dbaa9a6 100644 --- a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml @@ -61,7 +61,9 @@ initial_conditions: cldfrac_tot: 0.5 pbl_height: 25.0 phis : 0.1 - + dlf : 0.0 + dp_ccf : 0.0 + sh_ccf : 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From 604c2fa3c228ff7fd84b71daec008bddbca58f1d Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 27 Mar 2024 08:59:18 -0600 Subject: [PATCH 073/477] wet_scav - adding single-process. --- .../eamxx_mam_wetscav_process_interface.cpp | 6 +- .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../mam/wet_scav/CMakeLists.txt | 46 ++++++++++++ .../single-process/mam/wet_scav/input.yaml | 70 +++++++++++++++++++ .../mam/wet_scav/mam4_wetscav_standalone.cpp | 69 ++++++++++++++++++ .../single-process/mam/wet_scav/output.yaml | 14 ++++ 6 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/wet_scav/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp create mode 100644 components/eamxx/tests/single-process/mam/wet_scav/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 60f352f06e34..226dcb14d6e2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -64,6 +64,8 @@ void MAMWetscav::set_grids( // interfaces const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -77,6 +79,8 @@ void MAMWetscav::set_grids( "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice cloud water [kg/kg] wet + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure // -- Input variables that exists in PBUF in EAM static constexpr auto nondim = @@ -321,7 +325,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - + dry_atm_.p_int = get_field_in("p_int").get_view(); // How "buffer_" works: We use buffer to allocate memory for the members of // dry_atm_ object. Here we are providing those memory locations to the // dry_atm_ members. These members are computed from the above wet_atm_ or diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index c6b4e0748af3..3e1629ae0f36 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -21,6 +21,7 @@ if (SCREAM_ENABLE_MAM) # initial conditions. add_subdirectory(mam/optics) add_subdirectory(mam/aci) + add_subdirectory(mam/wet_scav) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt new file mode 100644 index 000000000000..dbddf4d26e29 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt @@ -0,0 +1,46 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_wetscav_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_wetscav physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} +) + +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 2 4 6) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_wetscav physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml new file mode 100644 index 000000000000..c73e2dbaa9a6 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -0,0 +1,70 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_wetscav] + mam_wetscav: + number_of_subcycles: ${NUM_SUBCYCLES} + compute_tendencies: [all] + lambda_low: 0.001 + +grids_manager: + Type: Mesh Free + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + cldn_prev_step: 0.5 + rprdsh: 1e-5 + rprddp: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + cldt_prev_step: 1e-5 + cldst: 1e-5 + qme: 1e-5 + prain: 1e-5 + evapr: 1e-5 + icwmrdp: 1e-5 + icwmrsh: 1e-5 + rprddp: 1e-5 + rprdsh: 1e-5 + sh_frac: 1e-5 + dp_frac: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + pbuf: 1e-5 # cloud-borne aerosols + dgncur_a: 1e-5 # aerosol particle diameter [m] + wetdens: 1e-5 # wet aerosol density [kg/m3] + qaerwat: 1e-5 # aerosol water [kg/kg] + dgnumwet: 1e-5 # wet aerosol diameter [m] + fracis: 1e-5 # fraction of transported species that are insoluble [fraction] + cldfrac_tot: 0.5 + pbl_height: 25.0 + phis : 0.1 + dlf : 0.0 + dp_ccf : 0.0 + sh_ccf : 0.0 +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp b/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp new file mode 100644 index 000000000000..203b51681d05 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp @@ -0,0 +1,69 @@ +#include + +#include "control/atmosphere_driver.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "physics/register_physics.hpp" +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/atm_process/atmosphere_process.hpp" + +#include "ekat/ekat_parse_yaml_file.hpp" +#include "ekat/logging/ekat_logger.hpp" + +#include + +namespace scream { + +TEST_CASE("mam4_wetscav-stand-alone", "") { + using namespace scream; + using namespace scream::control; + + // Create a comm + ekat::Comm atm_comm (MPI_COMM_WORLD); + + // Load ad parameter list + std::string fname = "input.yaml"; + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + logger.debug("yaml parsed."); + + // Time stepping parameters + const auto& ts = ad_params.sublist("time_stepping"); + const auto dt = ts.get("time_step"); + const auto nsteps = ts.get("number_of_steps"); + const auto t0_str = ts.get("run_t0"); + const auto t0 = util::str_to_time_stamp(t0_str); + + logger.info("running MAMWetscav standalone test with dt = {} for {} steps.", dt, nsteps); + EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); + + // Need to register products in the factory *before* we create any atm process or grids manager. + register_physics(); + register_mesh_free_grids_manager(); + register_diagnostics(); + logger.debug("products registered."); + + // Create the driver + AtmosphereDriver ad; + logger.debug("driver created."); + + // Init and run + ad.initialize(atm_comm,ad_params,t0); + logger.debug("driver initialized."); + + logger.info("Start time stepping loop ... [0%]"); + for (int i=0; i Date: Sat, 13 Apr 2024 20:16:45 -0600 Subject: [PATCH 074/477] wet_dep - update to match mam4xx interface. --- .../eamxx_mam_wetscav_process_interface.cpp | 122 ++++++++++++++---- .../eamxx_mam_wetscav_process_interface.hpp | 3 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 48 ++++++- 3 files changed, 147 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 226dcb14d6e2..be6e4d348cba 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -43,6 +43,9 @@ void MAMWetscav::set_grids( // Nevertheless, for output reasons, we like to see 'kg/kg'. auto q_unit = kg / kg; q_unit.set_string("kg/kg"); + // FIXME: units of tendencies; check units + auto dqdt_unit = kg / kg / s; + dqdt_unit.set_string("kg/kg/t"); auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); @@ -52,7 +55,7 @@ void MAMWetscav::set_grids( ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column - + const int nmodes = mam4::AeroConfig::num_modes();; // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -66,6 +69,9 @@ void MAMWetscav::set_grids( FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // Layout for 3D for nmodes + const FieldLayout scalar3d_layout_nmodes_mid{{COL, NMODES, LEV}, {ncol_, nmodes, nlev_}}; + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -180,13 +186,14 @@ void MAMWetscav::set_grids( // -- input/ouputs from PBUF for updating particle size and water uptake by // particles static constexpr auto m3 = m2 * m; - add_field("dgncur_a", scalar3d_layout_mid, m, - grid_name); // aerosol particle diameter [m] - add_field("wetdens", scalar3d_layout_mid, kg / m3, + add_field("dgncur_a", scalar3d_layout_nmodes_mid, m, + grid_name); // aerosol dry particle diameter [m] + add_field("wetdens", scalar3d_layout_nmodes_mid, kg / m3, grid_name); // wet aerosol density [kg/m3] - add_field("qaerwat", scalar3d_layout_mid, kg / kg, + add_field("qaerwat", scalar3d_layout_nmodes_mid, kg / kg, grid_name); // aerosol water [kg/kg] - add_field("dgnumwet", scalar3d_layout_mid, m, + // + add_field("dgnumwet", scalar3d_layout_nmodes_mid, m, grid_name); // wet aerosol diameter [m] add_field("fracis", scalar3d_layout_mid, nondim, grid_name); // fraction of transported species that are @@ -247,6 +254,33 @@ void MAMWetscav::set_grids( grid_name, "tracers"); } + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + std::string ptend_gas_name(mam_coupling::gas_mmr_field_name(g)); + ptend_gas_name += "ptend_"; + add_field(ptend_gas_name, scalar3d_layout_mid, dqdt_unit, + grid_name); + } + + // tendencies for interstitial aerosols + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + std::string ptend_num(mam_coupling::int_aero_nmr_field_name(imode)); + ptend_num += "ptend_"; + add_field(ptend_num, scalar3d_layout_mid, n_unit, + grid_name); + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + std::string ptend_int_mmr_field_name (int_mmr_field_name); + ptend_int_mmr_field_name += "ptend_"; + add_field(ptend_int_mmr_field_name, scalar3d_layout_mid, dqdt_unit, + grid_name); + } + } + } + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // Cloud fraction add_field("pbl_height", scalar2d_layout_mid, m, @@ -271,11 +305,11 @@ void MAMWetscav::set_grids( grid_name); // add_field("sh_ccf", scalar3d_layout_mid, n_unit, grid_name); // - // - add_field("aerdepwetis", scalar3d_layout_mid, n_unit, - grid_name); // - add_field("aerdepwetcw", scalar3d_layout_mid, n_unit, - grid_name); // + // FIXME: do we need to write aerdepwetis and aerdepwetcw? + // add_field("aerdepwetis", scalar3d_layout_mid, n_unit, + // grid_name); // + // add_field("aerdepwetcw", scalar3d_layout_mid, n_unit, + // grid_name); // } // ========================================================================================= @@ -349,7 +383,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.phis = get_field_in("phis").get_view(); - dry_atm_.z_surf = 0.0; // MUST FIXME: for now + dry_atm_.z_surf = 0.0; // ---- set wet/dry aerosol-related gas state data for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); @@ -391,6 +425,33 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } } + + // ---- set aerosol-related gas tendencies data + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + std::string ptend_mmr_field_name(mam_coupling::gas_mmr_field_name(g)); + ptend_mmr_field_name += "ptend_"; + dry_aero_tends_.gas_mmr[g] = get_field_out(ptend_mmr_field_name).get_view(); + } + + // set aerosol state tendencies data (interstitial aerosols only) + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + std::string ptend_int_nmr_field_name(mam_coupling::int_aero_nmr_field_name(imode)); + ptend_int_nmr_field_name +="ptend_"; + dry_aero_tends_.int_aero_nmr[imode] = + get_field_out(ptend_int_nmr_field_name).get_view(); + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + std::string ptend_int_nmr_field_name(int_mmr_field_name); + ptend_int_nmr_field_name +="ptend_"; + dry_aero_tends_.int_aero_mmr[imode][ispec] = + get_field_out(ptend_int_nmr_field_name).get_view(); + } + } + } + // set up our preprocess/postprocess functors // Here we initialize (not compute) objects in preprocess struct using the // objects in the argument list @@ -409,6 +470,8 @@ void MAMWetscav::initialize_impl(const RunType run_type) { const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); + aerdepwetis_ = view_2d("aerdepwetis", ncol_, mam4::aero_model::pcnst); + aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, mam4::aero_model::pcnst); } // ========================================================================================= @@ -454,6 +517,7 @@ void MAMWetscav::run_impl(const double dt) { const auto &dry_aero = dry_aero_; const auto &qqcw_sav = qqcw_sav_; const auto &work = work_; + const auto & dry_aero_tends= dry_aero_tends_; // inputs/outputs auto dlf = get_field_out("dlf").get_view(); @@ -485,8 +549,13 @@ void MAMWetscav::run_impl(const double dt) { .get_view(); // ?? // outputs - const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); - const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); + const auto aerdepwetis = aerdepwetis_;//get_field_out("aerdepwetis").get_view(); + const auto aerdepwetcw = aerdepwetcw_;// get_field_out("aerdepwetcw").get_view(); + + const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); + const auto dry_geometric_mean_diameter_i = get_field_out("dgncur_a").get_view(); + const auto qaerwat = get_field_out("qaerwat").get_view(); + const auto wetdens = get_field_out("wetdens").get_view(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); @@ -501,8 +570,8 @@ void MAMWetscav::run_impl(const double dt) { // fetch column-specific subviews into aerosol prognostics mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aero, icol); - // FIXME: we need to create tendecies... - mam4::Tendencies tends{}; + // fetch column-specific subviews into aerosol tendencies + mam4::Tendencies tends = mam_coupling::aerosols_tendencies_for_column(dry_aero_tends, icol); // shallow_convective_cloud_fraction auto cldn_prev_step_icol = ekat::subview(cldn_prev_step, icol); @@ -530,9 +599,9 @@ void MAMWetscav::run_impl(const double dt) { auto cldst_icol = ekat::subview(cldst, icol); // deep_convective_cloud_fraction; - auto dp_ccf_icol = ekat::subview(dp_ccf, icol); - // shallow_convective_cloud_fraction; - auto sh_ccf_icol = ekat::subview(sh_ccf, icol); + // auto dp_ccf_icol = ekat::subview(dp_ccf, icol); + // // shallow_convective_cloud_fraction; + // auto sh_ccf_icol = ekat::subview(sh_ccf, icol); // total_convective_detrainment; auto dlf_icol = ekat::subview(dlf, icol); //aerosol_wet_deposition_interstitial; @@ -541,18 +610,23 @@ void MAMWetscav::run_impl(const double dt) { auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol);\ auto work_icol = ekat::subview(work, icol); // auto qqcw_sav_icol = ekat::subview(qqcw_sav,icol); -#if 1 + auto wet_diameter_icol = ekat::subview(wet_geometric_mean_diameter_i,icol); + auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i,icol); + auto qaerwat_icol = ekat::subview(qaerwat,icol); + auto wetdens_icol = ekat::subview(wetdens,icol); + mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, // inputs - cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, - evapcsh_icol, dp_frac_icol, sh_frac_icol, dp_ccf_icol, sh_ccf_icol, - icwmrdp_col, icwmrsh_icol, evapr_icol, cldst_icol, + cldst_icol, cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, + evapcsh_icol, dp_frac_icol, sh_frac_icol, + icwmrdp_col, icwmrsh_icol, evapr_icol, dlf_icol, + wet_diameter_icol,dry_diameter_icol, + qaerwat_icol, wetdens_icol, // output aerdepwetis_icol, aerdepwetcw_icol, // FIXME remove qqcw_sav qqcw_sav, work_icol); -#endif }); // icol parallel_for loop /* diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index beb3d0c6789f..fc6886357858 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -181,9 +181,10 @@ class MAMWetscav : public scream::AtmosphereProcess { Kokkos::View qqcw_sav_; // aerosol states - mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; mam_coupling::Buffer buffer_; + mam_coupling::view_2d aerdepwetis_, aerdepwetcw_; std::shared_ptr m_grid; }; // class MAMWetscav diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 3a0011468826..99a116d4872e 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -562,6 +562,34 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, return progs; } + +// Given an AerosolState with views for dry aerosol quantities, creates a +// mam4::Tendencies object for the column with the given index with +// ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to +// mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Tendencies interstitial_aerosols_tendencies_for_column(const AerosolState& dry_aero, + const int column_index) { + constexpr int nlev = mam4::nlev; + mam4::Tendencies tends(nlev); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), + "int_aero_nmr not defined for dry aerosol state!"); + tends.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + tends.q_aero_i[m][a] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), + "gas_mmr not defined for dry aerosol state!"); + tends.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); + } + return tends; +} + // Given a dry aerosol state, creates a mam4::Prognostics object for the column // with the given index with interstitial and cloudborne aerosol views defined. // This object can be provided to mam4xx for the column. @@ -581,7 +609,25 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, } return progs; } - +// Given a dry aerosol state tendencies, creates a mam4::Tendencies object for the column +// with the given index with interstitial and cloudborne aerosol views defined. +// This object can be provided to mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Tendencies aerosols_tendencies_for_column(const AerosolState& dry_aero, + const int column_index) { + auto tends = interstitial_aerosols_tendencies_for_column(dry_aero, column_index); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), + "dry_cld_aero_nmr not defined for aerosol state!"); + tends.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.cld_aero_mmr[m][a].data()) { + tends.q_aero_c[m][a] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + } + } + } + return tends; +} // Given a thread team and a dry atmosphere state, dispatches threads from the // team to compute vertical layer heights and interfaces for the column with // the given index. From 1ac385d8c448dcefae71c4529799f1c9fd08695e Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sat, 13 Apr 2024 20:51:59 -0600 Subject: [PATCH 075/477] wet_dep - update mam4xx interface. --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 15 ++++----------- .../mam/eamxx_mam_wetscav_process_interface.hpp | 2 -- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index be6e4d348cba..7fa333f97bdb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -463,15 +463,10 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // wetdep constexpr int pcnst = mam4::aero_model::pcnst; - // FIXME: qqcw_sav_ should be part of work_ - // we need to add columns - qqcw_sav_ =Kokkos::View("qqcw_sav", mam4::nlev); - - const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); - aerdepwetis_ = view_2d("aerdepwetis", ncol_, mam4::aero_model::pcnst); - aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, mam4::aero_model::pcnst); + aerdepwetis_ = view_2d("aerdepwetis", ncol_, pcnst); + aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, pcnst); } // ========================================================================================= @@ -515,7 +510,6 @@ void MAMWetscav::run_impl(const double dt) { // We cannot use member of this class inside of the parallel_for const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; - const auto &qqcw_sav = qqcw_sav_; const auto &work = work_; const auto & dry_aero_tends= dry_aero_tends_; @@ -624,9 +618,8 @@ void MAMWetscav::run_impl(const double dt) { wet_diameter_icol,dry_diameter_icol, qaerwat_icol, wetdens_icol, // output - aerdepwetis_icol, aerdepwetcw_icol, - // FIXME remove qqcw_sav - qqcw_sav, work_icol); + aerdepwetis_icol, aerdepwetcw_icol, + work_icol); }); // icol parallel_for loop /* diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index fc6886357858..c09a789b6c7e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -178,8 +178,6 @@ class MAMWetscav : public scream::AtmosphereProcess { mam_coupling::DryAtmosphere dry_atm_; // wet dep view_2d work_; - Kokkos::View - qqcw_sav_; // aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; From c5c0cfab20ebe02a22eeab1aa0416b2ddba4adae Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sun, 14 Apr 2024 12:14:55 -0600 Subject: [PATCH 076/477] wet_dep - update name of tendencies. --- .../eamxx_mam_wetscav_process_interface.cpp | 34 ++++++++----------- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 7fa333f97bdb..1b8ee2d8e28f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -256,26 +256,23 @@ void MAMWetscav::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_gas_name(mam_coupling::gas_mmr_field_name(g)); - ptend_gas_name += "ptend_"; - add_field(ptend_gas_name, scalar3d_layout_mid, dqdt_unit, + std::string ptend_gas_name = "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); + add_field(ptend_gas_name, scalar3d_layout_mid, dqdt_unit, grid_name); } // tendencies for interstitial aerosols for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_num(mam_coupling::int_aero_nmr_field_name(imode)); - ptend_num += "ptend_"; - add_field(ptend_num, scalar3d_layout_mid, n_unit, + std::string ptend_num = "ptend_" +std::string(mam_coupling::int_aero_nmr_field_name(imode)); + add_field(ptend_num, scalar3d_layout_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_mmr_field_name (int_mmr_field_name); - ptend_int_mmr_field_name += "ptend_"; - add_field(ptend_int_mmr_field_name, scalar3d_layout_mid, dqdt_unit, + std::string ptend_int_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); + add_field(ptend_int_mmr_field_name, scalar3d_layout_mid, dqdt_unit, grid_name); } } @@ -428,15 +425,15 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // ---- set aerosol-related gas tendencies data for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_mmr_field_name(mam_coupling::gas_mmr_field_name(g)); - ptend_mmr_field_name += "ptend_"; + std::string ptend_mmr_field_name = "ptend_" + + std::string(mam_coupling::gas_mmr_field_name(g)); dry_aero_tends_.gas_mmr[g] = get_field_out(ptend_mmr_field_name).get_view(); } // set aerosol state tendencies data (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_int_nmr_field_name(mam_coupling::int_aero_nmr_field_name(imode)); - ptend_int_nmr_field_name +="ptend_"; + std::string ptend_int_nmr_field_name ="ptend_"+ + std::string(mam_coupling::int_aero_nmr_field_name(imode)); dry_aero_tends_.int_aero_nmr[imode] = get_field_out(ptend_int_nmr_field_name).get_view(); @@ -444,10 +441,9 @@ void MAMWetscav::initialize_impl(const RunType run_type) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_nmr_field_name(int_mmr_field_name); - ptend_int_nmr_field_name +="ptend_"; + std::string ptend_int_aero_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); dry_aero_tends_.int_aero_mmr[imode][ispec] = - get_field_out(ptend_int_nmr_field_name).get_view(); + get_field_out(ptend_int_aero_mmr_field_name).get_view(); } } } @@ -511,7 +507,7 @@ void MAMWetscav::run_impl(const double dt) { const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; const auto &work = work_; - const auto & dry_aero_tends= dry_aero_tends_; + const auto &dry_aero_tends= dry_aero_tends_; // inputs/outputs auto dlf = get_field_out("dlf").get_view(); @@ -565,8 +561,8 @@ void MAMWetscav::run_impl(const double dt) { mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aero, icol); // fetch column-specific subviews into aerosol tendencies - mam4::Tendencies tends = mam_coupling::aerosols_tendencies_for_column(dry_aero_tends, icol); - + // Note: we are only updating interstitial aerosols. + mam4::Tendencies tends = mam_coupling::interstitial_aerosols_tendencies_for_column(dry_aero_tends, icol); // shallow_convective_cloud_fraction auto cldn_prev_step_icol = ekat::subview(cldn_prev_step, icol); ///shallow_convective_precipitation_production diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 99a116d4872e..e48b5c9f2e1a 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -618,7 +618,7 @@ mam4::Tendencies aerosols_tendencies_for_column(const AerosolState& dry_aero, auto tends = interstitial_aerosols_tendencies_for_column(dry_aero, column_index); for (int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), - "dry_cld_aero_nmr not defined for aerosol state!"); + "Tendencies : dry_cld_aero_nmr not defined for aerosol state!"); tends.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.cld_aero_mmr[m][a].data()) { From 593c8b169a63f608d7403aafc3ca769a1bcd90f3 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 20 May 2024 14:16:18 -0600 Subject: [PATCH 077/477] wet_scav - rebase code --- .../eamxx_mam_wetscav_process_interface.cpp | 137 +++++++++--------- 1 file changed, 66 insertions(+), 71 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 1b8ee2d8e28f..123de4d95f65 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -55,37 +55,32 @@ void MAMWetscav::set_grids( ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column - const int nmodes = mam4::AeroConfig::num_modes();; - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; + const int nmodes = mam4::AeroConfig::num_modes(); - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and - // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + // layout for 3D (2d horiz X 1d vertical) variables at level midpoints/interfaces + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); - // Layout for 2D (2d horiz) variable defined at mid-level and - // interfaces - const FieldLayout scalar2d_layout_mid{{COL}, {ncol_}}; + // layout for 2D (1d horiz X 1d vertical) variables + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); - FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - - // Layout for 3D for nmodes - const FieldLayout scalar3d_layout_nmodes_mid{{COL, NMODES, LEV}, {ncol_, nmodes, nlev_}}; + // layout for 3D (ncol, nmodes, nlevs) + FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true,nmodes,"nmodes"); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, + add_field("T_mid", scalar3d_mid, K, grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, + add_field("p_mid", scalar3d_mid, Pa, grid_name); // pressure at mid points in [Pa - add_field("pseudo_density", scalar3d_layout_mid, Pa, + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); // pseudo density in [Pa] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); // ice cloud water [kg/kg] wet - add_field("p_int", scalar3d_layout_int, Pa, + add_field("p_int", scalar3d_int, Pa, grid_name); // total pressure // -- Input variables that exists in PBUF in EAM @@ -94,20 +89,20 @@ void MAMWetscav::set_grids( // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. - add_field("cldn_prev_step", scalar3d_layout_mid, nondim, + add_field("cldn_prev_step", scalar3d_mid, nondim, grid_name); // layer cloud fraction [fraction] - add_field("cldst", scalar3d_layout_mid, nondim, + add_field("cldst", scalar3d_mid, nondim, grid_name); //?? - add_field("rprdsh", scalar3d_layout_mid, kg / kg / s, + add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); // rain production, shallow convection // [kg/kg/s] //NOT updated add_field( - "rprddp", scalar3d_layout_mid, kg / kg / s, + "rprddp", scalar3d_mid, kg / kg / s, grid_name); // rain production, deep convection [kg/kg/s] //NOT updated - add_field("evapcsh", scalar3d_layout_mid, kg / kg / s, + add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of shallow convective // //NOT updated precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_layout_mid, kg / kg / s, + add_field("evapcdp", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of deep convective //NOT // updated precipitation >=0. [kg/kg/s] @@ -115,42 +110,42 @@ void MAMWetscav::set_grids( // "inputs" data structure // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. - add_field("cldt_prev_step", scalar3d_layout_mid, nondim, + add_field("cldt_prev_step", scalar3d_mid, nondim, grid_name); // total cloud fraction [fraction] add_field( - "qme", scalar3d_layout_mid, kg / kg / s, + "qme", scalar3d_mid, kg / kg / s, grid_name); // net condensation/evaporation of cloud water [kg/kg/s] - add_field("prain", scalar3d_layout_mid, kg / kg / s, + add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // stratiform rain production rate [kg/kg/s] add_field( - "evapr", scalar3d_layout_mid, kg / kg / s, + "evapr", scalar3d_mid, kg / kg / s, grid_name); // evaporation from stratiform rain [kg/kg/s] //NOT updated // -- Input variables that exists in PBUF in EAM (in wetdep.F90) - add_field("icwmrdp", scalar3d_layout_mid, kg / kg, + add_field("icwmrdp", scalar3d_mid, kg / kg, grid_name); // In cloud water mixing ratio, deep // convection [kg/kg] //NOT updated - add_field("icwmrsh", scalar3d_layout_mid, kg / kg, + add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); // In cloud water mixing ratio, shallow // convection [kg/kg] //NOT updated - add_field("rprddp", scalar3d_layout_mid, kg / kg / s, + add_field("rprddp", scalar3d_mid, kg / kg / s, grid_name); // Rain production, deep convection [kg/kg/s] add_field( - "sh_frac", scalar3d_layout_mid, nondim, + "sh_frac", scalar3d_mid, nondim, grid_name); // Shallow convective cloud fraction [fraction] //NOT updated add_field( - "dp_frac", scalar3d_layout_mid, nondim, + "dp_frac", scalar3d_mid, nondim, grid_name); // Deep convective cloud fraction [fraction] //NOT updated add_field( - "icwmrsh", scalar3d_layout_mid, nondim, + "icwmrsh", scalar3d_mid, nondim, grid_name); add_field( - "icwmrdp", scalar3d_layout_mid, nondim, + "icwmrdp", scalar3d_mid, nondim, grid_name); - add_field("evapcsh", scalar3d_layout_mid, kg / kg / s, + add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_layout_mid, kg / kg / s, + add_field("evapcdp", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of deep convective // precipitation >=0. [kg/kg/s] @@ -162,40 +157,40 @@ void MAMWetscav::set_grids( // for the land model static constexpr auto m2 = m * m; add_field( - "bcphiwet", scalar3d_layout_mid, kg / m2 / s, + "bcphiwet", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of hydrophilic black carbon [kg/m2/s] add_field( - "bcphidry", scalar3d_layout_mid, kg / m2 / s, + "bcphidry", scalar3d_mid, kg / m2 / s, grid_name); // dry deposition of hydrophilic black carbon [kg/m2/s] add_field( - "ocphiwet", scalar3d_layout_mid, kg / m2 / s, + "ocphiwet", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of hydrophilic organic carbon [kg/m2/s] add_field( - "ocphidry", scalar3d_layout_mid, kg / m2 / s, + "ocphidry", scalar3d_mid, kg / m2 / s, grid_name); // dry deposition of hydrophilic organic carbon [kg/m2/s] - add_field("dstwet1", scalar3d_layout_mid, kg / m2 / s, + add_field("dstwet1", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of dust (bin1) [kg/m2/s] - add_field("dstwet2", scalar3d_layout_mid, kg / m2 / s, + add_field("dstwet2", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of dust (bin2) [kg/m2/s] - add_field("dstwet3", scalar3d_layout_mid, kg / m2 / s, + add_field("dstwet3", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of dust (bin3) [kg/m2/s] - add_field("dstwet4", scalar3d_layout_mid, kg / m2 / s, + add_field("dstwet4", scalar3d_mid, kg / m2 / s, grid_name); // wet deposition of dust (bin4) [kg/m2/s] // -- input/ouputs from PBUF for updating particle size and water uptake by // particles static constexpr auto m3 = m2 * m; - add_field("dgncur_a", scalar3d_layout_nmodes_mid, m, + add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); // aerosol dry particle diameter [m] - add_field("wetdens", scalar3d_layout_nmodes_mid, kg / m3, + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); // wet aerosol density [kg/m3] - add_field("qaerwat", scalar3d_layout_nmodes_mid, kg / kg, + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); // aerosol water [kg/kg] // - add_field("dgnumwet", scalar3d_layout_nmodes_mid, m, + add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); // wet aerosol diameter [m] - add_field("fracis", scalar3d_layout_mid, nondim, + add_field("fracis", scalar3d_mid, nondim, grid_name); // fraction of transported species that are // insoluble [fraction] @@ -207,7 +202,7 @@ void MAMWetscav::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(imode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios @@ -216,7 +211,7 @@ void MAMWetscav::set_grids( // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { @@ -224,7 +219,7 @@ void MAMWetscav::set_grids( const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } @@ -234,7 +229,7 @@ void MAMWetscav::set_grids( if(strlen(cld_mmr_field_name) > 0) { // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these // are NOT advected - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } @@ -250,21 +245,21 @@ void MAMWetscav::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { std::string ptend_gas_name = "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); - add_field(ptend_gas_name, scalar3d_layout_mid, dqdt_unit, + add_field(ptend_gas_name, scalar3d_mid, dqdt_unit, grid_name); } // tendencies for interstitial aerosols for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { std::string ptend_num = "ptend_" +std::string(mam_coupling::int_aero_nmr_field_name(imode)); - add_field(ptend_num, scalar3d_layout_mid, n_unit, + add_field(ptend_num, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios @@ -272,40 +267,40 @@ void MAMWetscav::set_grids( mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { std::string ptend_int_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); - add_field(ptend_int_mmr_field_name, scalar3d_layout_mid, dqdt_unit, + add_field(ptend_int_mmr_field_name, scalar3d_mid, dqdt_unit, grid_name); } } } - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); // Cloud fraction - add_field("pbl_height", scalar2d_layout_mid, m, + add_field("pbl_height", scalar2d, m, grid_name); // PBL height static constexpr auto s2 = s * s; - add_field("phis", scalar2d_layout_mid, m2 / s2, + add_field("phis", scalar2d, m2 / s2, grid_name); // surface geopotential - add_field("qv", scalar3d_layout_mid, q_unit, + add_field("qv", scalar3d_mid, q_unit, grid_name); // specific humidity - add_field("nc", scalar3d_layout_mid, n_unit, + add_field("nc", scalar3d_mid, n_unit, grid_name); // cloud water number conc - add_field("ni", scalar3d_layout_mid, n_unit, + add_field("ni", scalar3d_mid, n_unit, grid_name); // cloud ice number conc - add_field("omega", scalar3d_layout_mid, Pa / s, + add_field("omega", scalar3d_mid, Pa / s, grid_name); // vertical pressure velocity // FIXME: units - add_field("dlf", scalar3d_layout_mid, n_unit, + add_field("dlf", scalar3d_mid, n_unit, grid_name); // - add_field("dp_ccf", scalar3d_layout_mid, n_unit, + add_field("dp_ccf", scalar3d_mid, n_unit, grid_name); // - add_field("sh_ccf", scalar3d_layout_mid, n_unit, + add_field("sh_ccf", scalar3d_mid, n_unit, grid_name); // // FIXME: do we need to write aerdepwetis and aerdepwetcw? - // add_field("aerdepwetis", scalar3d_layout_mid, n_unit, + // add_field("aerdepwetis", scalar3d_mid, n_unit, // grid_name); // - // add_field("aerdepwetcw", scalar3d_layout_mid, n_unit, + // add_field("aerdepwetcw", scalar3d_mid, n_unit, // grid_name); // } From a76689c251480912e9333078c993c147dd3fa8de Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 20 May 2024 17:06:21 -0600 Subject: [PATCH 078/477] wet_scav - adding missing input variable. --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 123de4d95f65..2f444f366782 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -142,6 +142,10 @@ void MAMWetscav::set_grids( add_field( "icwmrdp", scalar3d_mid, nondim, grid_name); + + add_field( + "prain", scalar3d_mid, nondim, + grid_name); add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] @@ -533,6 +537,9 @@ void MAMWetscav::run_impl(const double dt) { auto icwmrdp = get_field_out("icwmrdp") .get_view(); // ?? + auto prain = get_field_out("prain") + .get_view(); // ?? + // outputs const auto aerdepwetis = aerdepwetis_;//get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = aerdepwetcw_;// get_field_out("aerdepwetcw").get_view(); @@ -599,13 +606,15 @@ void MAMWetscav::run_impl(const double dt) { auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i,icol); auto qaerwat_icol = ekat::subview(qaerwat,icol); auto wetdens_icol = ekat::subview(wetdens,icol); + auto prain_icol = ekat::subview(prain,icol); + mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, // inputs cldst_icol, cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, - dlf_icol, + dlf_icol, prain_icol, wet_diameter_icol,dry_diameter_icol, qaerwat_icol, wetdens_icol, // output From 5cd64aef926b0a7633797e60068445cf78978e36 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 20 May 2024 20:52:16 -0600 Subject: [PATCH 079/477] wet_scav - removing old comments and variables that not needed. --- .../eamxx_mam_wetscav_process_interface.cpp | 371 +++--------------- 1 file changed, 46 insertions(+), 325 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 2f444f366782..9ac0720d8ab3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -2,9 +2,6 @@ #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" #include "scream_config.h" // for SCREAM_CIME_BUILD -// NOTE: see the impl/ directory for the contents of the impl namespace -#include "impl/compute_particle_size.cpp" - // Remove the following<<<< #include #include @@ -65,8 +62,9 @@ void MAMWetscav::set_grids( FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); // layout for 3D (ncol, nmodes, nlevs) - FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true,nmodes,"nmodes"); + FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); + // FieldLayout scalar2d = m_grid->get_2d_scalar_layout // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -89,9 +87,9 @@ void MAMWetscav::set_grids( // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. - add_field("cldn_prev_step", scalar3d_mid, nondim, + add_field("cldn", scalar3d_mid, nondim, grid_name); // layer cloud fraction [fraction] - add_field("cldst", scalar3d_mid, nondim, + add_field("cldt", scalar3d_mid, nondim, grid_name); //?? add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); // rain production, shallow convection @@ -110,11 +108,13 @@ void MAMWetscav::set_grids( // "inputs" data structure // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. - add_field("cldt_prev_step", scalar3d_mid, nondim, - grid_name); // total cloud fraction [fraction] - add_field( - "qme", scalar3d_mid, kg / kg / s, - grid_name); // net condensation/evaporation of cloud water [kg/kg/s] + // add_field("cldt_prev_step", scalar3d_mid, nondim, + // grid_name); // total cloud fraction [fraction] + + // FIXME: we do not need qme + // add_field( + // "qme", scalar3d_mid, kg / kg / s, + // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // stratiform rain production rate [kg/kg/s] add_field( @@ -128,30 +128,27 @@ void MAMWetscav::set_grids( add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); // In cloud water mixing ratio, shallow // convection [kg/kg] //NOT updated - add_field("rprddp", scalar3d_mid, kg / kg / s, - grid_name); // Rain production, deep convection [kg/kg/s] + // add_field("rprddp", scalar3d_mid, kg / kg / s, + // grid_name); // Rain production, deep convection [kg/kg/s] add_field( "sh_frac", scalar3d_mid, nondim, grid_name); // Shallow convective cloud fraction [fraction] //NOT updated add_field( "dp_frac", scalar3d_mid, nondim, grid_name); // Deep convective cloud fraction [fraction] //NOT updated + // in cloud water mixing ratio, deep shallow [kg/kg] add_field( "icwmrsh", scalar3d_mid, nondim, grid_name); - add_field( - "icwmrdp", scalar3d_mid, nondim, - grid_name); - - add_field( - "prain", scalar3d_mid, nondim, - grid_name); - add_field("evapcsh", scalar3d_mid, kg / kg / s, - grid_name); // Evaporation rate of shallow convective - // precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_mid, kg / kg / s, - grid_name); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] + // add_field( + // "icwmrdp", scalar3d_mid, nondim, + // grid_name); + // add_field("evapcsh", scalar3d_mid, kg / kg / s, + // grid_name); // Evaporation rate of shallow convective + // // precipitation >=0. [kg/kg/s] + // add_field("evapcdp", scalar3d_mid, kg / kg / s, + // grid_name); // Evaporation rate of deep convective + // // precipitation >=0. [kg/kg/s] // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process @@ -295,15 +292,17 @@ void MAMWetscav::set_grids( grid_name); // vertical pressure velocity // FIXME: units - add_field("dlf", scalar3d_mid, n_unit, - grid_name); // - add_field("dp_ccf", scalar3d_mid, n_unit, - grid_name); // - add_field("sh_ccf", scalar3d_mid, n_unit, + add_field("dlf", scalar3d_mid, kg / kg / s, grid_name); // + // add_field("dp_ccf", scalar3d_mid, n_unit, + // grid_name); // + // add_field("sh_ccf", scalar3d_mid, n_unit, + // grid_name); // + // FIXME: do we need to write aerdepwetis and aerdepwetcw? // add_field("aerdepwetis", scalar3d_mid, n_unit, // grid_name); // + // add_field("aerdepwetcw", scalar3d_mid, n_unit, // grid_name); // } @@ -460,7 +459,10 @@ void MAMWetscav::initialize_impl(const RunType run_type) { constexpr int pcnst = mam4::aero_model::pcnst; const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); + // FIXME: do I need to move these variables to FM? + // aerosol wet deposition (interstitial) [kg/m2/s] aerdepwetis_ = view_2d("aerdepwetis", ncol_, pcnst); + // aerosol wet deposition (cloud water) [kg/m2/s] aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, pcnst); } @@ -474,35 +476,6 @@ void MAMWetscav::run_impl(const double dt) { // needed by this process or setting up MAM4xx classes and their objects Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - - /*Fortran code: - call modal_aero_calcsize_sub(state%ncol, state%lchnk, state%q, state%pdel, - dt, & !in qqcw, ptend, dgnumdry_m=dgncur_a) !inout - - ----subroutine modal_aero_calcsize_sub(ncol, lchnk, state_q, pdel, deltat, - qqcw, ptend, do_adjust_in, & do_aitacc_transfer_in, list_idx_in, - update_mmr_in, dgnumdry_m) */ - // mam4::CalcSizeProcess process_(aero_config_); //initiate MAM4xx calcsize - // process - - /* ---------------------------------------------------------------------------------------- - * Compute particle size using the calcsize process - * ---------------------------------------------------------------------------------------- - */ - - // -- configure the process - /* - * -- NOTES: 1. Flags for the inter-mode particle transfer - * (do_aitacc_transfer) and size adjustment (do_adjust) are TRUE by default - * a. Size adjustment is only done by changing aerosol - * numbers in the modes. - * 2. Interstitial and cld borne aerosols (i.e. "tends") mmr will - * be updated (update_mmr is TRUE by default) - */ - // MUST: FIXME: Make sure state is not updated...only tendencies should be - // updated!! - // NOTE! we need a const mam_coupling::DryAtmosphere dry_atm for gpu access. - // We cannot use member of this class inside of the parallel_for const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; const auto &work = work_; @@ -510,14 +483,14 @@ void MAMWetscav::run_impl(const double dt) { // inputs/outputs auto dlf = get_field_out("dlf").get_view(); - auto dp_ccf = get_field_out("dp_ccf").get_view(); - auto sh_ccf = get_field_out("sh_ccf").get_view(); + // auto dp_ccf = get_field_out("dp_ccf").get_view(); + // auto sh_ccf = get_field_out("sh_ccf").get_view(); - auto cldn_prev_step = get_field_out("cldn_prev_step").get_view< Real **>(); + auto cldn = get_field_out("cldn").get_view< Real **>(); // where is cldt_prev_step used? auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - auto cldst = get_field_out("cldst").get_view< Real **>();//?? + auto cldt = get_field_out("cldt").get_view< Real **>();//?? auto evapr = get_field_out("evapr").get_view< Real **>(); auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow @@ -540,9 +513,11 @@ void MAMWetscav::run_impl(const double dt) { auto prain = get_field_out("prain") .get_view(); // ?? + const auto aerdepwetis = aerdepwetis_;// + const auto aerdepwetcw = aerdepwetcw_;// // outputs - const auto aerdepwetis = aerdepwetis_;//get_field_out("aerdepwetis").get_view(); - const auto aerdepwetcw = aerdepwetcw_;// get_field_out("aerdepwetcw").get_view(); + // const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); + // const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); const auto dry_geometric_mean_diameter_i = get_field_out("dgncur_a").get_view(); @@ -566,7 +541,7 @@ void MAMWetscav::run_impl(const double dt) { // Note: we are only updating interstitial aerosols. mam4::Tendencies tends = mam_coupling::interstitial_aerosols_tendencies_for_column(dry_aero_tends, icol); // shallow_convective_cloud_fraction - auto cldn_prev_step_icol = ekat::subview(cldn_prev_step, icol); + auto cldn_icol = ekat::subview(cldn, icol); ///shallow_convective_precipitation_production auto rprdsh_icol = ekat::subview(rprdsh, icol); // deep_convective_precipitation_production @@ -581,40 +556,28 @@ void MAMWetscav::run_impl(const double dt) { auto sh_frac_icol = ekat::subview(sh_frac, icol); // FIXME: what is this? - // diags.deep_convective_cloud_condensate auto icwmrdp_col = ekat::subview(icwmrdp, icol); - // diags.shallow_convective_cloud_condensate auto icwmrsh_icol = ekat::subview(icwmrsh, icol); - // diags.evaporation_of_falling_precipitation = auto evapr_icol = ekat::subview(evapr, icol); - // diags.stratiform_cloud_fraction = - auto cldst_icol = ekat::subview(cldst, icol); - - // deep_convective_cloud_fraction; - // auto dp_ccf_icol = ekat::subview(dp_ccf, icol); - // // shallow_convective_cloud_fraction; - // auto sh_ccf_icol = ekat::subview(sh_ccf, icol); - // total_convective_detrainment; + auto cldt_icol = ekat::subview(cldt, icol); + auto dlf_icol = ekat::subview(dlf, icol); - //aerosol_wet_deposition_interstitial; auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); - // diags.aerosol_wet_deposition_cloud_water; auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol);\ auto work_icol = ekat::subview(work, icol); - // auto qqcw_sav_icol = ekat::subview(qqcw_sav,icol); auto wet_diameter_icol = ekat::subview(wet_geometric_mean_diameter_i,icol); auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i,icol); auto qaerwat_icol = ekat::subview(qaerwat,icol); auto wetdens_icol = ekat::subview(wetdens,icol); auto prain_icol = ekat::subview(prain,icol); - mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, // inputs - cldst_icol, cldn_prev_step_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, + cldt_icol, cldn_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, dlf_icol, prain_icol, + // in/out wet_diameter_icol,dry_diameter_icol, qaerwat_icol, wetdens_icol, // output @@ -622,248 +585,6 @@ void MAMWetscav::run_impl(const double dt) { work_icol); }); // icol parallel_for loop - /* - call calc_sfc_flux(rprdsh(:ncol,:), state%pdel(:ncol,:), - rprdshsum(:ncol)) ! output the last argument call - calc_sfc_flux(rprddp(:ncol,:), state%pdel(:ncol,:), rprddpsum(:ncol)) ! - output the last argument call calc_sfc_flux(evapcsh(:ncol,:), - state%pdel(:ncol,:), evapcshsum(:ncol)) ! output the last argument call - calc_sfc_flux(evapcdp(:ncol,:), state%pdel(:ncol,:), evapcdpsum(:ncol)) ! - output the last argument - - ! initiate variables - qsrflx_mzaer2cnvpr(:,:,:) = 0.0_r8 - aerdepwetis(:,:) = 0.0_r8 - aerdepwetcw(:,:) = 0.0_r8 - qqcw_tmp(:,:) = 0.0_r8 - ! below-cloud scavcoef = 0.0 for cloud-borne species - scavcoefnv(:,:,0) = 0.0_r8 - ! resuspension goes to a different phase or mode - rtscavt_sv(:,:,:) = 0.0_r8 - - ! examine if there is precipitation falling from above in each grid - call examine_prec_exist ( ncol, state%pdel, & ! in - dep_inputs%prain, dep_inputs%cmfdqr,& ! in - dep_inputs%evapr, & ! in - isprx ) ! out - - ! calculate the mass-weighted sol_factic for coarse mode species - call set_f_act_coarse( ncol, & ! in - state%q, ptend%q, dt, & ! in - f_act_conv_coarse, f_act_conv_coarse_dust, & ! out - f_act_conv_coarse_nacl ) ! out - - mmode_loop_aa: do mtmp = 1, ntot_amode ! main loop over aerosol modes - - ! for mam4, do accum, aitken, pcarbon, then coarse - ! so change the order of 3 and 4 here - imode = mode_order_change(mtmp) - - ! loop over interstitial (1) and cloud-borne (2) forms - !BSINGH (09/12/2014):Do cloudborne first for unified convection scheme - so !that the resuspension of cloudborne can be saved then applied to - interstitial (RCE) lphase_loop_aa: do lphase = 2,1,-1 ! do cloudborne (2) - first then interstitial (1) - - if (lphase == 1) then ! interstial aerosol - call modal_aero_bcscavcoef_get( imode, ncol, isprx, dgnumwet, & - scavcoefnv(:,:,1), scavcoefnv(:,:,2) ) - endif - call define_act_frac ( lphase, imode, & ! in - sol_facti, sol_factic, sol_factb, f_act_conv) ! out - - ! REASTER 08/12/2015 - changed ordering (mass then number) for - prevap resuspend to coarse lspec_loop_aa: do lspec = 1, - nspec_amode(imode)+2 ! loop over number + chem constituents + water - - call index_ordering ( & - lspec, imode, lphase, & ! in - jaeronumb, jaeromass, jaerowater, & ! in - mm, jnv, jnummaswtr ) ! out - - if (mm <= 0 .or. jnummaswtr == jaerowater ) cycle ! by pass wet - aerosols - - ! mam_prevap_resusp_optcc values control the prevap_resusp - calculations in wetdepa_v2: ! 0 = no resuspension ! 130 = non-linear - resuspension of aerosol mass based on scavenged aerosol mass ! 230 = - non-linear resuspension of aerosol number based on raindrop number ! the - 130 thru 230 all use the new prevap_resusp code block in subr wetdepa_v2 - ! - mam_prevap_resusp_optcc = mam_prevap_resusp_no - - if ( jnummaswtr == jaeromass ) then ! dry mass - mam_prevap_resusp_optcc = mam_prevap_resusp_mass - elseif ( jnummaswtr == jaeronumb .and. lphase == 1 .and. imode == - modeptr_coarse ) then ! number mam_prevap_resusp_optcc = - mam_prevap_resusp_num endif - - ! set f_act_conv for interstitial (lphase=1) coarse mode species - ! for the convective in-cloud, we conceptually treat the coarse - dust and seasalt ! as being externally mixed, and apply ! f_act_conv = - f_act_conv_coarse_dust/nacl to dust/seasalt ! number and sulfate are - conceptually partitioned to the dust and seasalt ! on a mass basis, so the - f_act_conv for number and sulfate are ! mass-weighted averages of the - values used for dust/seasalt if ((lphase == 1) .and. (imode == - modeptr_coarse)) then f_act_conv = f_act_conv_coarse if (jnummaswtr == - jaeromass) then if (lmassptr_amode(lspec,imode) == - lptr_dust_a_amode(imode)) then f_act_conv = f_act_conv_coarse_dust elseif - (lmassptr_amode(lspec,imode) == lptr_nacl_a_amode(imode)) then f_act_conv = - f_act_conv_coarse_nacl endif endif endif - - lphase_jnmw_conditional: if (lphase == 1) then - ptend%lq(mm) = .true. - ! q_tmp reflects changes from modal_aero_calcsize and is the - "most current" q q_tmp(1:ncol,:) = state%q(1:ncol,:,mm) + - ptend%q(1:ncol,:,mm)*dt !Feed in the saved cloudborne mixing ratios from - phase 2 qqcw_in(:,:) = qqcw_sav(:,:,lspec) - - call wetdepa_v2( & - ncol, dt, state%pdel, & ! in dep_inputs%cmfdqr, - dep_inputs%evapc, dlf, dep_inputs%conicw, & ! in dep_inputs%prain, - dep_inputs%evapr, dep_inputs%totcond, & ! in dep_inputs%cldt, - dep_inputs%cldcu, & ! in dep_inputs%cldvcu, - dep_inputs%cldvst, & ! in sol_factb, sol_facti, - sol_factic, & ! in mam_prevap_resusp_optcc, - .false., scavcoefnv(:,:,jnv), f_act_conv, & ! in q_tmp, qqcw_in(:,:), & ! - in fracis(:,:,mm), dqdt_tmp, iscavt, & ! out - icscavt, isscavt, bcscavt, bsscavt, rcscavt, rsscavt ) ! - out - - ! resuspension goes to coarse mode - call calc_resusp_to_coarse( ncol, mm, & ! in - mmtoo_prevap_resusp, .true., & ! in - rcscavt, rsscavt, & ! in - dqdt_tmp, rtscavt_sv ) ! inout - - ptend%q(1:ncol,:,mm) = ptend%q(1:ncol,:,mm) + - dqdt_tmp(1:ncol,:) - - call outfld( trim(cnst_name(mm))//'WET', dqdt_tmp(:,:), pcols, - lchnk) call outfld( trim(cnst_name(mm))//'SIC', icscavt, pcols, lchnk) call - outfld( trim(cnst_name(mm))//'SIS', isscavt, pcols, lchnk) call outfld( - trim(cnst_name(mm))//'SBC', bcscavt, pcols, lchnk) call outfld( - trim(cnst_name(mm))//'SBS', bsscavt, pcols, lchnk) - - call calc_sfc_flux(dqdt_tmp(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx aerdepwetis(:ncol,mm) = sflx(:ncol) - - call calc_sfc_flux(icscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx sflxic = sflx - - call calc_sfc_flux(isscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name(mm))//'SFSIS', sflx, - pcols, lchnk) - - call calc_sfc_flux(bcscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name(mm))//'SFSBC', sflx, - pcols, lchnk) sflxbc = sflx - - call calc_sfc_flux(bsscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name(mm))//'SFSBS', sflx, - pcols, lchnk) - - ! here the prevap resuspension is in rcscavt & rsscavt and - column integral is written to history !BSINGH(09/15/2014):Following two - nested do-loops are new additions for unified convection - !BSINGH(09/15/2014):After these do-loops, code was added by - RCE, the comments by RCE are kept as it is call - calc_sfc_flux(rcscavt(:ncol,:), state%pdel(:ncol,:), sflx(:ncol)) ! output - sflx sflxec = sflx - - call calc_sfc_flux(rsscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name(mm))//'SFSES', sflx, - pcols, lchnk) - - ! apportion convective surface fluxes to deep and shallow conv - ! this could be done more accurately in subr wetdepa - ! since deep and shallow rarely occur simultaneously, and - these ! fields are just diagnostics, this approximate method is adequate - ! only do this for interstitial aerosol, because conv clouds - to not ! affect the stratiform-cloudborne aerosol call - apportion_sfc_flux_deep ( ncol, & ! in - rprddpsum,rprdshsum,evapcdpsum,evapcshsum,& ! in - sflxbc, sflxec, & ! in - sflxbcdp, sflxecdp ) ! out - - call outfld( trim(cnst_name(mm))//'SFSBD', sflxbcdp, pcols, - lchnk) ! when ma_convproc_intr is used, convective in-cloud wet removal is - done there ! the convective (total and deep) precip-evap-resuspension - includes in- and below-cloud ! contributions, so pass the below-cloud - contribution to ma_convproc_intr qsrflx_mzaer2cnvpr(1:ncol,mm,1) = sflxec( - 1:ncol) qsrflx_mzaer2cnvpr(1:ncol,mm,2) = sflxecdp(1:ncol) - - elseif (lphase == 2) then lphase_jnmw_conditional - ! There is no cloud-borne aerosol water in the model, so this - code block ! should NEVER execute for lspec = nspec_amode(m)+1 (i.e., - jnummaswtr = 2). ! The code only worked because the "do lspec" loop cycles - when lspec = nspec_amode(m)+1, ! but that does not make the code correct. - fldcw => qqcw_get_field(pbuf,mm,lchnk) - qqcw_sav(1:ncol,:,lspec) = fldcw(1:ncol,:) !RCE 2012/01/12 - - ! FIXME: Not sure if this is a bug or not as qqcw_tmp seem - different ! from the previous call and qqcw_tmp is always zero. May need ! - further check. - Shuaiqi Tang in refactoring for MAM4xx call wetdepa_v2( & - ncol, dt, state%pdel, & ! in dep_inputs%cmfdqr, - dep_inputs%evapc, dlf, dep_inputs%conicw, & ! in dep_inputs%prain, - dep_inputs%evapr, dep_inputs%totcond, & ! in dep_inputs%cldt, - dep_inputs%cldcu, & ! in dep_inputs%cldvcu, - dep_inputs%cldvst, & ! in sol_factb, sol_facti, - sol_factic, & ! in mam_prevap_resusp_optcc, - .true., scavcoefnv(:,:,jnv), f_act_conv, & ! in fldcw, qqcw_tmp, & ! in - fracis_cw, dqdt_tmp, iscavt, & ! out icscavt, isscavt, - bcscavt, bsscavt, rcscavt, rsscavt ) ! out - - ! resuspension goes to coarse mode - call calc_resusp_to_coarse( ncol, mm, & ! in - mmtoo_prevap_resusp, .false., & ! in - rcscavt, rsscavt, & ! in - dqdt_tmp, rtscavt_sv ) ! inout - - fldcw(1:ncol,:) = fldcw(1:ncol,:) + dqdt_tmp(1:ncol,:) * dt - - call calc_sfc_flux(dqdt_tmp(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFWET', - sflx, pcols, lchnk) aerdepwetcw(:ncol,mm) = sflx(:ncol) - - call calc_sfc_flux(icscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSIC', - sflx, pcols, lchnk) - - call calc_sfc_flux(isscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSIS', - sflx, pcols, lchnk) - - call calc_sfc_flux(bcscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSBC', - sflx, pcols, lchnk) - - call calc_sfc_flux(bsscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSBS', - sflx, pcols, lchnk) - - call calc_sfc_flux(rcscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSEC', - sflx, pcols, lchnk) - - call calc_sfc_flux(rsscavt(:ncol,:), state%pdel(:ncol,:), - sflx(:ncol)) ! output sflx call outfld( trim(cnst_name_cw(mm))//'SFSES', - sflx, pcols, lchnk) - - endif lphase_jnmw_conditional - - enddo lspec_loop_aa ! lspec = 1, nspec_amode(m)+2 - enddo lphase_loop_aa ! lphase = 1, 2 - enddo mmode_loop_aa ! m = 1, ntot_amode - - ! if the user has specified prescribed aerosol dep fluxes then - ! do not set cam_out dep fluxes according to the prognostic aerosols - if (.not.aerodep_flx_prescribed()) then - call set_srf_wetdep(aerdepwetis, aerdepwetcw, cam_out) - endif - - call wetdep_inputs_unset(dep_inputs) - */ std::cout << "End of wetscav run" << std::endl; } From 711199bac4b1870f284fcc6c0029fda6d9d09ea9 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 23 May 2024 14:26:34 -0600 Subject: [PATCH 080/477] wet_scav - updating code after rebase. --- .../physics/mam/eamxx_mam_wetscav_process_interface.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 9ac0720d8ab3..a3cc31338d8b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -39,13 +39,8 @@ void MAMWetscav::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. auto q_unit = kg / kg; - q_unit.set_string("kg/kg"); - // FIXME: units of tendencies; check units auto dqdt_unit = kg / kg / s; - dqdt_unit.set_string("kg/kg/t"); - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); m_grid = grids_manager->get_grid("Physics"); const auto &grid_name = m_grid->name(); @@ -115,7 +110,7 @@ void MAMWetscav::set_grids( // add_field( // "qme", scalar3d_mid, kg / kg / s, // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] - add_field("prain", scalar3d_mid, kg / kg / s, + add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // stratiform rain production rate [kg/kg/s] add_field( "evapr", scalar3d_mid, kg / kg / s, @@ -489,7 +484,7 @@ void MAMWetscav::run_impl(const double dt) { auto cldn = get_field_out("cldn").get_view< Real **>(); // where is cldt_prev_step used? - auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? + // auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? auto cldt = get_field_out("cldt").get_view< Real **>();//?? auto evapr = get_field_out("evapr").get_view< Real **>(); auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow From 1b8052812bc2fa34c0ee02b898a0bd80f13fbc97 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 23 May 2024 16:06:35 -0600 Subject: [PATCH 081/477] wet_scav - Adding missing variables in input file. --- components/eamxx/tests/single-process/mam/wet_scav/input.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index c73e2dbaa9a6..02290fed670b 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -27,11 +27,13 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} cldn_prev_step: 0.5 + cldn: 0.5 rprdsh: 1e-5 rprddp: 1e-5 evapcsh: 1e-5 evapcdp: 1e-5 cldt_prev_step: 1e-5 + cldt: 1e-5 cldst: 1e-5 qme: 1e-5 prain: 1e-5 From 8ec139cc95a0440ab6bd43ac896d36dba3b50b4c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 09:04:48 -0600 Subject: [PATCH 082/477] wet_scav - Moving 'aerdepwetis' and 'aerdepwetcw' to the field manager and adding them to the output.yaml file. --- .../eamxx_mam_wetscav_process_interface.cpp | 33 ++++++++----------- .../single-process/mam/wet_scav/output.yaml | 6 ++-- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index a3cc31338d8b..5542ff23d709 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -47,7 +47,8 @@ void MAMWetscav::set_grids( ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column - const int nmodes = mam4::AeroConfig::num_modes(); + const int nmodes = mam4::AeroConfig::num_modes(); // Number of modes + constexpr int pcnst = mam4::aero_model::pcnst; // layout for 3D (2d horiz X 1d vertical) variables at level midpoints/interfaces FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); @@ -59,7 +60,8 @@ void MAMWetscav::set_grids( // layout for 3D (ncol, nmodes, nlevs) FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); - // FieldLayout scalar2d = m_grid->get_2d_scalar_layout + // layout for 2D (ncol, pcnst) + FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "pcnst"); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -293,13 +295,12 @@ void MAMWetscav::set_grids( // grid_name); // // add_field("sh_ccf", scalar3d_mid, n_unit, // grid_name); // - - // FIXME: do we need to write aerdepwetis and aerdepwetcw? - // add_field("aerdepwetis", scalar3d_mid, n_unit, - // grid_name); // - - // add_field("aerdepwetcw", scalar3d_mid, n_unit, - // grid_name); // + // aerosol wet deposition (interstitial) + add_field("aerdepwetis", scalar2d_pconst, kg/m2/s, + grid_name); // + // aerosol wet deposition (cloud water) + add_field("aerdepwetcw", scalar2d_pconst, kg/m2/s, + grid_name); // } // ========================================================================================= @@ -451,14 +452,9 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_aero_); // wetdep - constexpr int pcnst = mam4::aero_model::pcnst; + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); - // FIXME: do I need to move these variables to FM? - // aerosol wet deposition (interstitial) [kg/m2/s] - aerdepwetis_ = view_2d("aerdepwetis", ncol_, pcnst); - // aerosol wet deposition (cloud water) [kg/m2/s] - aerdepwetcw_ = view_2d("aerdepwetcw", ncol_, pcnst); } // ========================================================================================= @@ -507,12 +503,9 @@ void MAMWetscav::run_impl(const double dt) { auto prain = get_field_out("prain") .get_view(); // ?? - - const auto aerdepwetis = aerdepwetis_;// - const auto aerdepwetcw = aerdepwetcw_;// // outputs - // const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); - // const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); + const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); + const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); const auto dry_geometric_mean_diameter_i = get_field_out("dgncur_a").get_view(); diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 3dc34fdac75d..c63885d069d3 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -5,10 +5,10 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid - + - aerdepwetis + - aerdepwetcw output_control: - Frequency: 2 + Frequency: 2 frequency_units: nsteps MPI Ranks in Filename: true ... From 7ced79ab1557e90210d36ec1997be3037e0019d7 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 09:11:05 -0600 Subject: [PATCH 083/477] wet_scav - Removing headers that are part of mam_coupling and unused views. --- .../mam/eamxx_mam_wetscav_process_interface.hpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index c09a789b6c7e..6f15752953cd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -7,15 +7,6 @@ // For declaring wetscav class derived from atm process class #include "share/atm_process/atmosphere_process.hpp" -// For MAM4 processes -#include - -// For MAM4 calcsize process (FIXME:should we include it in mam4 or mam_coupling??) -#include - -// For wetdep processes -#include - // For component name #include @@ -182,7 +173,6 @@ class MAMWetscav : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; mam_coupling::Buffer buffer_; - mam_coupling::view_2d aerdepwetis_, aerdepwetcw_; std::shared_ptr m_grid; }; // class MAMWetscav From 6b6a79d7be4526d6d7a1376d4faddc4d4ef96c36 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 09:17:16 -0600 Subject: [PATCH 084/477] wet_scav - Adding output fields. --- .../eamxx/tests/single-process/mam/wet_scav/output.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index c63885d069d3..e9ad22d7119f 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -7,6 +7,10 @@ Fields: Field Names: - aerdepwetis - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat output_control: Frequency: 2 frequency_units: nsteps From d39659a7d960f22fcc70b4f227806b16c83e92d0 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 09:54:48 -0600 Subject: [PATCH 085/477] wet_scav - Adding outputs and running test for 3 steps. --- .../single-process/mam/wet_scav/input.yaml | 8 +-- .../single-process/mam/wet_scav/output.yaml | 52 ++++++++++++++++++- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 02290fed670b..698966f0559b 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -6,7 +6,7 @@ driver_options: time_stepping: time_step: ${ATM_TIME_STEP} run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} + number_of_steps: 3 atmosphere_processes: atm_procs_list: [mam4_wetscav] @@ -26,8 +26,8 @@ grids_manager: initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - cldn_prev_step: 0.5 - cldn: 0.5 + cldn_prev_step: 0.5 + cldn: 0.5 rprdsh: 1e-5 rprddp: 1e-5 evapcsh: 1e-5 @@ -35,7 +35,7 @@ initial_conditions: cldt_prev_step: 1e-5 cldt: 1e-5 cldst: 1e-5 - qme: 1e-5 + qme: 1e-5 prain: 1e-5 evapr: 1e-5 icwmrdp: 1e-5 diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index e9ad22d7119f..4912603943d2 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -11,8 +11,58 @@ Fields: - dgncur_a - wetdens - qaerwat + - ptend_bc_a1 + - ptend_bc_a3 + - ptend_bc_a4 + - ptend_dst_a1 + - ptend_dst_a3 + - ptend_mom_a1 + - ptend_mom_a2 + - ptend_mom_a3 + - ptend_mom_a4 + - ptend_nacl_a1 + - ptend_nacl_a2 + - ptend_nacl_a3 + - ptend_num_a1 + - ptend_num_a2 + - ptend_num_a3 + - ptend_num_a4 + - ptend_pom_a1 + - ptend_pom_a3 + - ptend_pom_a4 + - ptend_so4_a1 + - ptend_so4_a2 + - ptend_so4_a3 + - ptend_soa_a1 + - ptend_soa_a2 + - ptend_soa_a3 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - pom_c1 + - pom_c3 + - pom_c4 + - so4_c1 + - so4_c2 + - so4_c3 + - soa_c1 + - soa_c2 + - soa_c3 output_control: - Frequency: 2 + Frequency: 1 frequency_units: nsteps MPI Ranks in Filename: true ... From 6e81baa54c22b4f6d4d8ebed15ada3fb6882f7ed Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 10:03:54 -0600 Subject: [PATCH 086/477] wet_sav- removin unused input. --- components/eamxx/tests/single-process/mam/wet_scav/input.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 698966f0559b..bf7fe8e5380e 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -54,7 +54,6 @@ initial_conditions: dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] - pbuf: 1e-5 # cloud-borne aerosols dgncur_a: 1e-5 # aerosol particle diameter [m] wetdens: 1e-5 # wet aerosol density [kg/m3] qaerwat: 1e-5 # aerosol water [kg/kg] From 5b0b24399625a6f9f5c25e4537db031e8e512d00 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 10:05:59 -0600 Subject: [PATCH 087/477] wet_sav - Adding FIXME. --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 5542ff23d709..037c6d17acb5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -150,7 +150,7 @@ void MAMWetscav::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- - + // FIXME: we have not added code to update the surface fluxes. // -- surface fluxes (input/outpts) for the coupler's cam_out data struture // for the land model static constexpr auto m2 = m * m; From 6e553176938fff232725166274a5b2745bbe416c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 10:11:42 -0600 Subject: [PATCH 088/477] wet_sav - Updating comments. --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 037c6d17acb5..418e8b3bf267 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -11,8 +11,6 @@ Future work: Wirte comments write in/outs for all variables clearly - - */ namespace scream { @@ -474,9 +472,6 @@ void MAMWetscav::run_impl(const double dt) { // inputs/outputs auto dlf = get_field_out("dlf").get_view(); - // auto dp_ccf = get_field_out("dp_ccf").get_view(); - // auto sh_ccf = get_field_out("sh_ccf").get_view(); - auto cldn = get_field_out("cldn").get_view< Real **>(); // where is cldt_prev_step used? @@ -496,13 +491,12 @@ void MAMWetscav::run_impl(const double dt) { .get_view(); // Deep convective cloud fraction [fraction] auto icwmrsh = get_field_out("icwmrsh") - .get_view(); // ?? - + .get_view(); //in cloud water mixing ratio, shallow convection auto icwmrdp = get_field_out("icwmrdp") - .get_view(); // ?? + .get_view(); // in cloud water mixing ratio, deep convection auto prain = get_field_out("prain") - .get_view(); // ?? + .get_view(); // stratiform rain production rate // outputs const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); From 8cd04d2c33b50f0a74d1886881a38faef970c391 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 10:25:10 -0600 Subject: [PATCH 089/477] wet_scav - Setting Frequency=2 --- components/eamxx/tests/single-process/mam/wet_scav/input.yaml | 2 +- components/eamxx/tests/single-process/mam/wet_scav/output.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index bf7fe8e5380e..c8def99dd627 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -6,7 +6,7 @@ driver_options: time_stepping: time_step: ${ATM_TIME_STEP} run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: 3 + number_of_steps: ${NUM_STEPS} atmosphere_processes: atm_procs_list: [mam4_wetscav] diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 4912603943d2..13493a002483 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -62,7 +62,7 @@ Fields: - soa_c2 - soa_c3 output_control: - Frequency: 1 + Frequency: 2 frequency_units: nsteps MPI Ranks in Filename: true ... From e91b2e25fe4ccb4769700025038efbf791707916 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 11:40:17 -0600 Subject: [PATCH 090/477] wet_scav - Adding multi-process test for mam4_wetsav. --- .../dynamics_physics/CMakeLists.txt | 2 +- .../CMakeLists.txt | 95 ++++++++++ .../input.yaml | 107 ++++++++++++ .../output.yaml | 164 ++++++++++++++++++ 4 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml diff --git a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 64ac2187c280..b9b58f2fe86f 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -13,7 +13,7 @@ if (SCREAM_DOUBLE_PRECISION) # initial conditions. #add_subdirectory(homme_mam4xx_pg2) add_subdirectory(mam/homme_shoc_cld_p3_mam_optics_rrtmgp) + add_subdirectory(mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav) endif() endif() endif() - diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..41a915b92656 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt @@ -0,0 +1,95 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 41) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction shoc spa p3 scream_rrtmgp mam ${dynLibName} + LABELS dynamics shoc cld p3 rrtmgp physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 2) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 1) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/init/spa_init_ne2np4.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc + scream/init/${EAMxx_tests_IC_FILE_72lev} + cam/topo/${EAMxx_tests_TOPO_FILE} +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp spa mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml new file mode 100644 index 000000000000..bfbec956d852 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -0,0 +1,107 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_liq_surf_mass: 0.0 + precip_ice_surf_mass: 0.0 + cldn: 0.5 + rprdsh: 1e-5 + rprddp: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + cldt: 1e-5 + prain: 1e-5 + evapr: 1e-5 + icwmrdp: 1e-5 + icwmrsh: 1e-5 + rprddp: 1e-5 + rprdsh: 1e-5 + sh_frac: 1e-5 + dp_frac: 1e-5 + evapcsh: 1e-5 + evapcdp: 1e-5 + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + dgncur_a: 1e-5 # aerosol particle diameter [m] + wetdens: 1e-5 # wet aerosol density [kg/m3] + qaerwat: 1e-5 # aerosol water [kg/kg] + dgnumwet: 1e-5 # wet aerosol diameter [m] + fracis: 1e-5 # fraction of transported species that are insoluble [fraction] + cldfrac_tot: 0.5 + pbl_height: 25.0 + phis : 0.1 + dlf : 0.0 + dp_ccf : 0.0 + sh_ccf : 0.0 + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mac_aero_mic,rrtmgp] + schedule_type: Sequential + Type: Group + mac_aero_mic: + atm_procs_list: [shoc,CldFraction,spa,p3,mam4_wetscav] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + spa: + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc + p3: + max_total_ni: 740.0e3 + shoc: + check_flux_state_consistency: true + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + mam_wetscav: + number_of_subcycles: ${NUM_SUBCYCLES} + compute_tendencies: [all] + lambda_low: 0.001 + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml new file mode 100644 index 000000000000..f85225d1274f --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml @@ -0,0 +1,164 @@ +%YAML 1.1 +--- +filename_prefix: homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav_output +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + # SHOC + - cldfrac_liq + - eddy_diff_mom + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # SPA + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + - nccn + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + HOMME + - horiz_winds + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + HOMME + - T_mid + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + # Diagnostics + - T_mid_at_lev_2 + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_500mb + - T_mid_at_500hPa + - T_mid_at_50000Pa + - PotentialTemperature + - AtmosphereDensity + - Exner + - VirtualTemperature + - z_int + - geopotential_int_at_lev_2 + - z_mid_at_500mb + - geopotential_mid + - dz + - DryStaticEnergy + - SeaLevelPressure + - LiqWaterPath + - IceWaterPath + - VapWaterPath + - RainWaterPath + - RimeWaterPath + - ShortwaveCloudForcing + - LongwaveCloudForcing + - RelativeHumidity + - ZonalVapFlux + - MeridionalVapFlux + - PotentialTemperature_at_model_top + - PotentialTemperature_at_500mb + # MAM4_WETSCAV + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat + - ptend_bc_a1 + - ptend_bc_a3 + - ptend_bc_a4 + - ptend_dst_a1 + - ptend_dst_a3 + - ptend_mom_a1 + - ptend_mom_a2 + - ptend_mom_a3 + - ptend_mom_a4 + - ptend_nacl_a1 + - ptend_nacl_a2 + - ptend_nacl_a3 + - ptend_num_a1 + - ptend_num_a2 + - ptend_num_a3 + - ptend_num_a4 + - ptend_pom_a1 + - ptend_pom_a3 + - ptend_pom_a4 + - ptend_so4_a1 + - ptend_so4_a2 + - ptend_so4_a3 + - ptend_soa_a1 + - ptend_soa_a2 + - ptend_soa_a3 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - pom_c1 + - pom_c3 + - pom_c4 + - so4_c1 + - so4_c2 + - so4_c3 + - soa_c1 + - soa_c2 + - soa_c3 + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 2570f0613d383e2bfad5f375dfae29093651dfe0 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 27 May 2024 13:11:10 -0600 Subject: [PATCH 091/477] wet_sav - Adding wetscav to namelist. --- cime_config/tests.py | 1 + .../cime_config/namelist_defaults_scream.xml | 62 +++++++++++++++++++ .../scream/mam4xx/wetscav/shell_commands | 7 +++ 3 files changed, 70 insertions(+) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands diff --git a/cime_config/tests.py b/cime_config/tests.py index 06c16ca176c9..b4fb91cc5ee1 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -724,6 +724,7 @@ "tests" : ( "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", ) }, diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50188236e4e0..66036956f920 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -261,6 +261,10 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/poly_rrtmg_c20240206.nc + + + + @@ -530,6 +534,64 @@ be lost if SCREAM_HACK_XML is not enabled. 0 0.001 900.0 + + + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 0.5 + 25.0 + 0.1 + 0.0 + 0.0 + 0.0 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands new file mode 100644 index 000000000000..821feb10b7d7 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -0,0 +1,7 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,spa,p3,mam4_wetscav" -b From 7b1471db26a6c0c29d4a861b82f9d29bbbc13a0c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sat, 1 Jun 2024 09:36:13 -0600 Subject: [PATCH 092/477] wet_scav - Removing initial values of pbl_height and phis from namelist_defaults_scream.xml. Upgrading MAM4xx. --- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 66036956f920..79fcc64f3848 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -587,8 +587,6 @@ be lost if SCREAM_HACK_XML is not enabled. 1e-5 1e-5 0.5 - 25.0 - 0.1 0.0 0.0 0.0 From 25d4a916f9778707662c7d17286cacbf09eec65d Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sat, 1 Jun 2024 12:07:03 -0600 Subject: [PATCH 093/477] wet_scav - Adding line in CMakeLists. --- .../eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt index dbddf4d26e29..0624c8ed2020 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt @@ -7,7 +7,8 @@ CreateADUnitTest(${TEST_BASE_NAME} LABELS mam4_wetscav physics LIBS mam MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} -) + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} + ) # Set AD configurable options SetVarDependingOnTestSize(NUM_STEPS 2 4 6) From 3e6bf35a2bb579a68c4e9a69fd4038e81ed38094 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Sun, 2 Jun 2024 10:08:25 -0600 Subject: [PATCH 094/477] wet_scav - Update interstitial aerosol state. --- .../eamxx_mam_wetscav_process_interface.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 418e8b3bf267..0fa9d1d4f252 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -459,8 +459,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - - printf("Working on wet_sav \n"); // preprocess input -- needs a scan for the calculation of all variables // needed by this process or setting up MAM4xx classes and their objects Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -509,6 +507,9 @@ void MAMWetscav::run_impl(const double dt) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + // Making a local copy of 'nlev_' because we cannot use a member of a class inside a parallel_for. + const int nlev= nlev_; + // loop over atmosphere columns and compute aerosol particle size Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { @@ -565,9 +566,20 @@ void MAMWetscav::run_impl(const double dt) { // output aerdepwetis_icol, aerdepwetcw_icol, work_icol); + team.team_barrier(); + // update interstitial aerosol state + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + const auto q_aero_i = progs.q_aero_i[m][a]; + const auto tends_q_aero_i =tends.q_aero_i[m][a]; + q_aero_i(kk) += tends_q_aero_i(kk) * dt; + } + } + }); + }); // icol parallel_for loop - std::cout << "End of wetscav run" << std::endl; } // ========================================================================================= From a55b7a78144ca774328f526f7119975aa61a5986 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 3 Jun 2024 19:54:35 -0700 Subject: [PATCH 095/477] A working CIME case with wetscav in the process list at the right place --- .../testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands index 821feb10b7d7..f0e96c413985 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -4,4 +4,4 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,spa,p3,mam4_wetscav" -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_wetscav,rrtmgp" -b From 7299556aab7b1bab086632ed9f7ee741b526e383 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 4 Jun 2024 14:09:48 -0700 Subject: [PATCH 096/477] Fixes single process test --- .../mam/wet_scav/CMakeLists.txt | 20 +++--- .../mam/wet_scav/mam4_wetscav_standalone.cpp | 69 ------------------- .../single-process/mam/wet_scav/output.yaml | 3 +- 3 files changed, 9 insertions(+), 83 deletions(-) delete mode 100644 components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp diff --git a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt index 0624c8ed2020..3881df3e059d 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt @@ -2,38 +2,34 @@ include (ScreamUtils) set (TEST_BASE_NAME mam4_wetscav_standalone) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + # Create the test CreateADUnitTest(${TEST_BASE_NAME} LABELS mam4_wetscav physics LIBS mam MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} - ) +) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 2 4 6) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) set (ATM_TIME_STEP 1800) set (RUN_T0 2021-10-12-45000) +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) -# Ensure test input files are present in the data dir -set (TEST_INPUT_FILES - scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc -) -foreach (file IN ITEMS ${TEST_INPUT_FILES}) - GetInputFile(${file}) -endforeach() - # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} - FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} LABELS mam4_wetscav physics META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 @@ -42,6 +38,6 @@ CompareNCFilesFamilyMpi ( if (SCREAM_ENABLE_BASELINE_TESTS) # Compare one of the output files with the baselines. # Note: one is enough, since we already check that np1 is BFB with npX - set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) endif() diff --git a/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp b/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp deleted file mode 100644 index 203b51681d05..000000000000 --- a/components/eamxx/tests/single-process/mam/wet_scav/mam4_wetscav_standalone.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/register_physics.hpp" -#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/logging/ekat_logger.hpp" - -#include - -namespace scream { - -TEST_CASE("mam4_wetscav-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - logger.debug("yaml parsed."); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - logger.info("running MAMWetscav standalone test with dt = {} for {} steps.", dt, nsteps); - EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); - - // Need to register products in the factory *before* we create any atm process or grids manager. - register_physics(); - register_mesh_free_grids_manager(); - register_diagnostics(); - logger.debug("products registered."); - - // Create the driver - AtmosphereDriver ad; - logger.debug("driver created."); - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - logger.debug("driver initialized."); - - logger.info("Start time stepping loop ... [0%]"); - for (int i=0; i Date: Tue, 4 Jun 2024 17:22:27 -0700 Subject: [PATCH 097/477] Fixes file for the mam4xx standalone tests --- .../tests/single-process/mam/wet_scav/input.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index c8def99dd627..f1e0194106a2 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -10,21 +10,20 @@ time_stepping: atmosphere_processes: atm_procs_list: [mam4_wetscav] - mam_wetscav: - number_of_subcycles: ${NUM_SUBCYCLES} - compute_tendencies: [all] - lambda_low: 0.001 grids_manager: Type: Mesh Free - grids_names: [Physics] - Physics: + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: type: point_grid + aliases: [Physics] number_of_global_columns: 218 number_of_vertical_levels: 72 initial_conditions: - Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} cldn_prev_step: 0.5 cldn: 0.5 From f65792df7fb381c141acc85b8b78759caa4f28c7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 10 Jun 2024 05:02:12 -0700 Subject: [PATCH 098/477] Removing uncoupled test files since they are renamed as single process tests --- .../uncoupled/mam4_wet_scav/CMakeLists.txt | 30 -------- .../tests/uncoupled/mam4_wet_scav/input.yaml | 70 ------------------- .../mam4_wet_scav/mam4_wetscav_standalone.cpp | 69 ------------------ .../tests/uncoupled/mam4_wet_scav/output.yaml | 14 ---- 4 files changed, 183 deletions(-) delete mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt delete mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml delete mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp delete mode 100644 components/eamxx/tests/uncoupled/mam4_wet_scav/output.yaml diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt deleted file mode 100644 index 8558f6fbb056..000000000000 --- a/components/eamxx/tests/uncoupled/mam4_wet_scav/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -include (ScreamUtils) - -# This test needs to be reworked for microphysics -- currently it's still using -# input for nucleation. - -# Create the test -CreateADUnitTest(mam4_wetscav_standalone - LABELS mam4_wetscav physics - LIBS mam - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} -) - -# Set AD configurable options -set (ATM_TIME_STEP 1) -SetVarDependingOnTestSize(NUM_STEPS 2 2 2) -set (RUN_T0 2021-10-12-45000) - -# ## Copy (and configure) yaml files needed by tests -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) - -# Ensure test input files are present in the data dir -set (TEST_INPUT_FILES - scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc -) -foreach (file IN ITEMS ${TEST_INPUT_FILES}) - GetInputFile(${file}) -endforeach() diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml b/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml deleted file mode 100644 index c73e2dbaa9a6..000000000000 --- a/components/eamxx/tests/uncoupled/mam4_wet_scav/input.yaml +++ /dev/null @@ -1,70 +0,0 @@ -%YAML 1.1 ---- -driver_options: - atmosphere_dag_verbosity_level: 5 - -time_stepping: - time_step: ${ATM_TIME_STEP} - run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} - -atmosphere_processes: - atm_procs_list: [mam4_wetscav] - mam_wetscav: - number_of_subcycles: ${NUM_SUBCYCLES} - compute_tendencies: [all] - lambda_low: 0.001 - -grids_manager: - Type: Mesh Free - grids_names: [Physics] - Physics: - type: point_grid - number_of_global_columns: 218 - number_of_vertical_levels: 72 - -initial_conditions: - Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc - topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - cldn_prev_step: 0.5 - rprdsh: 1e-5 - rprddp: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 - cldt_prev_step: 1e-5 - cldst: 1e-5 - qme: 1e-5 - prain: 1e-5 - evapr: 1e-5 - icwmrdp: 1e-5 - icwmrsh: 1e-5 - rprddp: 1e-5 - rprdsh: 1e-5 - sh_frac: 1e-5 - dp_frac: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] - pbuf: 1e-5 # cloud-borne aerosols - dgncur_a: 1e-5 # aerosol particle diameter [m] - wetdens: 1e-5 # wet aerosol density [kg/m3] - qaerwat: 1e-5 # aerosol water [kg/kg] - dgnumwet: 1e-5 # wet aerosol diameter [m] - fracis: 1e-5 # fraction of transported species that are insoluble [fraction] - cldfrac_tot: 0.5 - pbl_height: 25.0 - phis : 0.1 - dlf : 0.0 - dp_ccf : 0.0 - sh_ccf : 0.0 -# The parameters for I/O control -Scorpio: - output_yaml_files: ["output.yaml"] -... diff --git a/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp b/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp deleted file mode 100644 index 203b51681d05..000000000000 --- a/components/eamxx/tests/uncoupled/mam4_wet_scav/mam4_wetscav_standalone.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/register_physics.hpp" -#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/atm_process/atmosphere_process.hpp" - -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/logging/ekat_logger.hpp" - -#include - -namespace scream { - -TEST_CASE("mam4_wetscav-stand-alone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Load ad parameter list - std::string fname = "input.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - logger.debug("yaml parsed."); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - logger.info("running MAMWetscav standalone test with dt = {} for {} steps.", dt, nsteps); - EKAT_ASSERT_MSG (dt>0, "Error! Time step must be positive.\n"); - - // Need to register products in the factory *before* we create any atm process or grids manager. - register_physics(); - register_mesh_free_grids_manager(); - register_diagnostics(); - logger.debug("products registered."); - - // Create the driver - AtmosphereDriver ad; - logger.debug("driver created."); - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - logger.debug("driver initialized."); - - logger.info("Start time stepping loop ... [0%]"); - for (int i=0; i Date: Thu, 20 Jun 2024 14:42:13 -0700 Subject: [PATCH 099/477] Clang format and makes dp_frac and input, updates mam4xx submodule to point to a branch --- .../eamxx_mam_wetscav_process_interface.cpp | 242 ++++++++++-------- 1 file changed, 131 insertions(+), 111 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 0fa9d1d4f252..9656ac4e2fb6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -1,5 +1,7 @@ -#include #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +#include + #include "scream_config.h" // for SCREAM_CIME_BUILD // Remove the following<<<< @@ -36,19 +38,20 @@ void MAMWetscav::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; + auto q_unit = kg / kg; auto dqdt_unit = kg / kg / s; - auto n_unit = 1 / kg; // units of number mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers m_grid = grids_manager->get_grid("Physics"); const auto &grid_name = m_grid->name(); ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column - const int nmodes = mam4::AeroConfig::num_modes(); // Number of modes + const int nmodes = mam4::AeroConfig::num_modes(); // Number of modes constexpr int pcnst = mam4::aero_model::pcnst; - // layout for 3D (2d horiz X 1d vertical) variables at level midpoints/interfaces + // layout for 3D (2d horiz X 1d vertical) variables at level + // midpoints/interfaces FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); @@ -56,7 +59,8 @@ void MAMWetscav::set_grids( FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); // layout for 3D (ncol, nmodes, nlevs) - FieldLayout scalar3d_mid_nmodes = m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); + FieldLayout scalar3d_mid_nmodes = + m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); // layout for 2D (ncol, pcnst) FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "pcnst"); @@ -80,12 +84,15 @@ void MAMWetscav::set_grids( static constexpr auto nondim = Units::nondimensional(); // for variables that are fractions etc. + // Deep convective cloud fraction [fraction] //NOT updated + add_field("dp_frac", scalar3d_mid, nondim, grid_name); + // MUST FIXME: cldt and cldn are the same variables. They must be their // previous step values. add_field("cldn", scalar3d_mid, nondim, - grid_name); // layer cloud fraction [fraction] + grid_name); // layer cloud fraction [fraction] add_field("cldt", scalar3d_mid, nondim, - grid_name); //?? + grid_name); //?? add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); // rain production, shallow convection // [kg/kg/s] //NOT updated @@ -111,7 +118,7 @@ void MAMWetscav::set_grids( // "qme", scalar3d_mid, kg / kg / s, // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] add_field("prain", scalar3d_mid, kg / kg / s, - grid_name); // stratiform rain production rate [kg/kg/s] + grid_name); // stratiform rain production rate [kg/kg/s] add_field( "evapr", scalar3d_mid, kg / kg / s, grid_name); // evaporation from stratiform rain [kg/kg/s] //NOT updated @@ -124,17 +131,14 @@ void MAMWetscav::set_grids( grid_name); // In cloud water mixing ratio, shallow // convection [kg/kg] //NOT updated // add_field("rprddp", scalar3d_mid, kg / kg / s, - // grid_name); // Rain production, deep convection [kg/kg/s] + // grid_name); // Rain production, deep convection + // [kg/kg/s] add_field( "sh_frac", scalar3d_mid, nondim, grid_name); // Shallow convective cloud fraction [fraction] //NOT updated - add_field( - "dp_frac", scalar3d_mid, nondim, - grid_name); // Deep convective cloud fraction [fraction] //NOT updated + // in cloud water mixing ratio, deep shallow [kg/kg] - add_field( - "icwmrsh", scalar3d_mid, nondim, - grid_name); + add_field("icwmrsh", scalar3d_mid, nondim, grid_name); // add_field( // "icwmrdp", scalar3d_mid, nondim, // grid_name); @@ -198,8 +202,8 @@ void MAMWetscav::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(imode); - add_field(int_nmr_field_name, scalar3d_mid, n_unit, - grid_name, "tracers"); + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios const char *cld_nmr_field_name = @@ -207,16 +211,15 @@ void MAMWetscav::set_grids( // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - add_field(cld_nmr_field_name, scalar3d_mid, n_unit, - grid_name); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit, - grid_name, "tracers"); + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -225,8 +228,7 @@ void MAMWetscav::set_grids( if(strlen(cld_mmr_field_name) > 0) { // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these // are NOT advected - add_field(cld_mmr_field_name, scalar3d_mid, q_unit, - grid_name); + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } } @@ -241,30 +243,31 @@ void MAMWetscav::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, - grid_name, "tracers"); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); } // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_gas_name = "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); - add_field(ptend_gas_name, scalar3d_mid, dqdt_unit, - grid_name); + std::string ptend_gas_name = + "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); + add_field(ptend_gas_name, scalar3d_mid, dqdt_unit, grid_name); } // tendencies for interstitial aerosols for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_num = "ptend_" +std::string(mam_coupling::int_aero_nmr_field_name(imode)); - add_field(ptend_num, scalar3d_mid, n_unit, - grid_name); + std::string ptend_num = + "ptend_" + std::string(mam_coupling::int_aero_nmr_field_name(imode)); + add_field(ptend_num, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); + std::string ptend_int_mmr_field_name = + "ptend_" + std::string(int_mmr_field_name); add_field(ptend_int_mmr_field_name, scalar3d_mid, dqdt_unit, - grid_name); + grid_name); } } } @@ -288,16 +291,16 @@ void MAMWetscav::set_grids( // FIXME: units add_field("dlf", scalar3d_mid, kg / kg / s, - grid_name); // + grid_name); // // add_field("dp_ccf", scalar3d_mid, n_unit, // grid_name); // // add_field("sh_ccf", scalar3d_mid, n_unit, // grid_name); // // aerosol wet deposition (interstitial) - add_field("aerdepwetis", scalar2d_pconst, kg/m2/s, + add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); // // aerosol wet deposition (cloud water) - add_field("aerdepwetcw", scalar2d_pconst, kg/m2/s, + add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); // } @@ -417,15 +420,16 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // ---- set aerosol-related gas tendencies data for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_mmr_field_name = "ptend_" + - std::string(mam_coupling::gas_mmr_field_name(g)); - dry_aero_tends_.gas_mmr[g] = get_field_out(ptend_mmr_field_name).get_view(); + std::string ptend_mmr_field_name = + "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); + dry_aero_tends_.gas_mmr[g] = + get_field_out(ptend_mmr_field_name).get_view(); } - // set aerosol state tendencies data (interstitial aerosols only) + // set aerosol state tendencies data (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_int_nmr_field_name ="ptend_"+ - std::string(mam_coupling::int_aero_nmr_field_name(imode)); + std::string ptend_int_nmr_field_name = + "ptend_" + std::string(mam_coupling::int_aero_nmr_field_name(imode)); dry_aero_tends_.int_aero_nmr[imode] = get_field_out(ptend_int_nmr_field_name).get_view(); @@ -433,7 +437,8 @@ void MAMWetscav::initialize_impl(const RunType run_type) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(imode, ispec); if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_aero_mmr_field_name = "ptend_" + std::string(int_mmr_field_name); + std::string ptend_int_aero_mmr_field_name = + "ptend_" + std::string(int_mmr_field_name); dry_aero_tends_.int_aero_mmr[imode][ispec] = get_field_out(ptend_int_aero_mmr_field_name).get_view(); } @@ -452,7 +457,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // wetdep const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); - work_ = view_2d("work", ncol_, work_len); + work_ = view_2d("work", ncol_, work_len); } // ========================================================================================= @@ -465,50 +470,65 @@ void MAMWetscav::run_impl(const double dt) { Kokkos::fence(); const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; - const auto &work = work_; - const auto &dry_aero_tends= dry_aero_tends_; + const auto &work = work_; + const auto &dry_aero_tends = dry_aero_tends_; // inputs/outputs - auto dlf = get_field_out("dlf").get_view(); - auto cldn = get_field_out("cldn").get_view< Real **>(); + auto dlf = get_field_out("dlf").get_view(); + auto cldn = get_field_out("cldn").get_view(); // where is cldt_prev_step used? - // auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); //FIXME: Is it same as cldn_prev_step?? - auto cldt = get_field_out("cldt").get_view< Real **>();//?? - auto evapr = get_field_out("evapr").get_view< Real **>(); - auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow - - // convection [kg/kg/s] - auto evapcsh = get_field_out("evapcsh").get_view(); // Evaporation rate of shallow convective + // auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); + // //FIXME: Is it same as cldn_prev_step?? + auto cldt = get_field_out("cldt").get_view(); //?? + auto evapr = get_field_out("evapr").get_view(); + auto rprdsh = + get_field_out("rprdsh").get_view(); // rain production, shallow + + // convection [kg/kg/s] + auto evapcsh = + get_field_out("evapcsh") + .get_view(); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] - auto sh_frac = get_field_out("sh_frac").get_view(); // Shallow convective cloud fraction [fraction] - auto rprddp = get_field_out("rprddp").get_view(); // rain production, deep convection [kg/kg/s] - auto evapcdp = get_field_out("evapcdp").get_view(); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] - auto dp_frac = get_field_out("dp_frac") - .get_view(); // Deep convective cloud fraction [fraction] + auto sh_frac = + get_field_out("sh_frac") + .get_view(); // Shallow convective cloud fraction [fraction] + auto rprddp = + get_field_out("rprddp") + .get_view(); // rain production, deep convection [kg/kg/s] + auto evapcdp = + get_field_out("evapcdp") + .get_view(); // Evaporation rate of deep convective + // precipitation >=0. [kg/kg/s] + // Deep convective cloud fraction [fraction] + auto dp_frac = get_field_in("dp_frac").get_view(); auto icwmrsh = get_field_out("icwmrsh") - .get_view(); //in cloud water mixing ratio, shallow convection - auto icwmrdp = get_field_out("icwmrdp") - .get_view(); // in cloud water mixing ratio, deep convection + .get_view(); // in cloud water mixing ratio, + // shallow convection + auto icwmrdp = + get_field_out("icwmrdp") + .get_view(); // in cloud water mixing ratio, deep convection auto prain = get_field_out("prain") - .get_view(); // stratiform rain production rate + .get_view(); // stratiform rain production rate // outputs const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); - const auto wet_geometric_mean_diameter_i = get_field_out("dgnumwet").get_view(); - const auto dry_geometric_mean_diameter_i = get_field_out("dgncur_a").get_view(); + const auto wet_geometric_mean_diameter_i = + get_field_out("dgnumwet").get_view(); + const auto dry_geometric_mean_diameter_i = + get_field_out("dgncur_a").get_view(); const auto qaerwat = get_field_out("qaerwat").get_view(); const auto wetdens = get_field_out("wetdens").get_view(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - // Making a local copy of 'nlev_' because we cannot use a member of a class inside a parallel_for. - const int nlev= nlev_; + // Making a local copy of 'nlev_' because we cannot use a member of a class + // inside a parallel_for. + const int nlev = nlev_; // loop over atmosphere columns and compute aerosol particle size Kokkos::parallel_for( @@ -522,15 +542,17 @@ void MAMWetscav::run_impl(const double dt) { mam_coupling::aerosols_for_column(dry_aero, icol); // fetch column-specific subviews into aerosol tendencies // Note: we are only updating interstitial aerosols. - mam4::Tendencies tends = mam_coupling::interstitial_aerosols_tendencies_for_column(dry_aero_tends, icol); + mam4::Tendencies tends = + mam_coupling::interstitial_aerosols_tendencies_for_column( + dry_aero_tends, icol); // shallow_convective_cloud_fraction auto cldn_icol = ekat::subview(cldn, icol); - ///shallow_convective_precipitation_production + /// shallow_convective_precipitation_production auto rprdsh_icol = ekat::subview(rprdsh, icol); // deep_convective_precipitation_production auto rprddp_icol = ekat::subview(rprddp, icol); // deep_convective_precipitation_evaporation - auto evapcdp_icol = ekat::subview(evapcdp, icol); + auto evapcdp_icol = ekat::subview(evapcdp, icol); // shallow_convective_precipitation_evaporation = auto evapcsh_icol = ekat::subview(evapcsh, icol); // deep_convective_cloud_fraction @@ -539,47 +561,45 @@ void MAMWetscav::run_impl(const double dt) { auto sh_frac_icol = ekat::subview(sh_frac, icol); // FIXME: what is this? - auto icwmrdp_col = ekat::subview(icwmrdp, icol); + auto icwmrdp_col = ekat::subview(icwmrdp, icol); auto icwmrsh_icol = ekat::subview(icwmrsh, icol); - auto evapr_icol = ekat::subview(evapr, icol); - auto cldt_icol = ekat::subview(cldt, icol); - - auto dlf_icol = ekat::subview(dlf, icol); - auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); - auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol);\ - auto work_icol = ekat::subview(work, icol); - auto wet_diameter_icol = ekat::subview(wet_geometric_mean_diameter_i,icol); - auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i,icol); - auto qaerwat_icol = ekat::subview(qaerwat,icol); - auto wetdens_icol = ekat::subview(wetdens,icol); - auto prain_icol = ekat::subview(prain,icol); - - mam4::wetdep::aero_model_wetdep(team, atm, progs, tends, dt, - // inputs - cldt_icol, cldn_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, - evapcsh_icol, dp_frac_icol, sh_frac_icol, - icwmrdp_col, icwmrsh_icol, evapr_icol, - dlf_icol, prain_icol, - // in/out - wet_diameter_icol,dry_diameter_icol, - qaerwat_icol, wetdens_icol, - // output - aerdepwetis_icol, aerdepwetcw_icol, - work_icol); - team.team_barrier(); - // update interstitial aerosol state - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + auto evapr_icol = ekat::subview(evapr, icol); + auto cldt_icol = ekat::subview(cldt, icol); + + auto dlf_icol = ekat::subview(dlf, icol); + auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); + auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol); + auto work_icol = ekat::subview(work, icol); + auto wet_diameter_icol = + ekat::subview(wet_geometric_mean_diameter_i, icol); + auto dry_diameter_icol = + ekat::subview(dry_geometric_mean_diameter_i, icol); + auto qaerwat_icol = ekat::subview(qaerwat, icol); + auto wetdens_icol = ekat::subview(wetdens, icol); + auto prain_icol = ekat::subview(prain, icol); + + mam4::wetdep::aero_model_wetdep( + team, atm, progs, tends, dt, + // inputs + cldt_icol, cldn_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, + evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, + evapr_icol, dlf_icol, prain_icol, + // in/out + wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, + // output + aerdepwetis_icol, aerdepwetcw_icol, work_icol); + team.team_barrier(); + // update interstitial aerosol state + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - for(int a = 0; a < mam4::num_species_mode(m); ++a) { - const auto q_aero_i = progs.q_aero_i[m][a]; - const auto tends_q_aero_i =tends.q_aero_i[m][a]; - q_aero_i(kk) += tends_q_aero_i(kk) * dt; - } - } - }); - + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + const auto q_aero_i = progs.q_aero_i[m][a]; + const auto tends_q_aero_i = tends.q_aero_i[m][a]; + q_aero_i(kk) += tends_q_aero_i(kk) * dt; + } + } + }); }); // icol parallel_for loop - } // ========================================================================================= From 14efd0d666759460bc440474964e275d478f713f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 20 Jun 2024 19:55:15 -0700 Subject: [PATCH 100/477] Makes cldt intent-in (include changes to the mam4xx submodule) --- .../eamxx_mam_wetscav_process_interface.cpp | 56 ++++++++----------- .../single-process/mam/wet_scav/input.yaml | 8 +-- 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 9656ac4e2fb6..e71bda90ca94 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -81,18 +81,18 @@ void MAMWetscav::set_grids( grid_name); // total pressure // -- Input variables that exists in PBUF in EAM - static constexpr auto nondim = - Units::nondimensional(); // for variables that are fractions etc. + // for variables that are fractions etc. + static constexpr auto nondim = Units::nondimensional(); - // Deep convective cloud fraction [fraction] //NOT updated + // Deep convective cloud fraction [fraction] add_field("dp_frac", scalar3d_mid, nondim, grid_name); - // MUST FIXME: cldt and cldn are the same variables. They must be their - // previous step values. - add_field("cldn", scalar3d_mid, nondim, - grid_name); // layer cloud fraction [fraction] - add_field("cldt", scalar3d_mid, nondim, - grid_name); //?? + // Shallow convective cloud fraction [fraction] //NOT updated + add_field("sh_frac", scalar3d_mid, nondim, grid_name); + + // Total cloud fraction [fraction] + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); + add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); // rain production, shallow convection // [kg/kg/s] //NOT updated @@ -108,10 +108,6 @@ void MAMWetscav::set_grids( // -- Input variables that exists in PBUF in EAM (in wetdep.F90) in the // "inputs" data structure - // MUST FIXME: cldt and cldn are the same variables. They must be their - // previous step values. - // add_field("cldt_prev_step", scalar3d_mid, nondim, - // grid_name); // total cloud fraction [fraction] // FIXME: we do not need qme // add_field( @@ -133,9 +129,6 @@ void MAMWetscav::set_grids( // add_field("rprddp", scalar3d_mid, kg / kg / s, // grid_name); // Rain production, deep convection // [kg/kg/s] - add_field( - "sh_frac", scalar3d_mid, nondim, - grid_name); // Shallow convective cloud fraction [fraction] //NOT updated // in cloud water mixing ratio, deep shallow [kg/kg] add_field("icwmrsh", scalar3d_mid, nondim, grid_name); @@ -473,14 +466,18 @@ void MAMWetscav::run_impl(const double dt) { const auto &work = work_; const auto &dry_aero_tends = dry_aero_tends_; - // inputs/outputs - auto dlf = get_field_out("dlf").get_view(); - auto cldn = get_field_out("cldn").get_view(); + // inputs + // Shallow convective cloud fraction [fraction] + auto sh_frac = get_field_in("sh_frac").get_view(); + + // Deep convective cloud fraction [fraction] + auto dp_frac = get_field_in("dp_frac").get_view(); + + // Total cloud fraction + auto cldt = get_field_in("cldfrac_liq").get_view(); //?? - // where is cldt_prev_step used? - // auto cldt_prev_step = get_field_out("cldt_prev_step").get_view< Real **>(); - // //FIXME: Is it same as cldn_prev_step?? - auto cldt = get_field_out("cldt").get_view(); //?? + // inputs/outputs + auto dlf = get_field_out("dlf").get_view(); auto evapr = get_field_out("evapr").get_view(); auto rprdsh = get_field_out("rprdsh").get_view(); // rain production, shallow @@ -490,9 +487,6 @@ void MAMWetscav::run_impl(const double dt) { get_field_out("evapcsh") .get_view(); // Evaporation rate of shallow convective // precipitation >=0. [kg/kg/s] - auto sh_frac = - get_field_out("sh_frac") - .get_view(); // Shallow convective cloud fraction [fraction] auto rprddp = get_field_out("rprddp") .get_view(); // rain production, deep convection [kg/kg/s] @@ -500,8 +494,6 @@ void MAMWetscav::run_impl(const double dt) { get_field_out("evapcdp") .get_view(); // Evaporation rate of deep convective // precipitation >=0. [kg/kg/s] - // Deep convective cloud fraction [fraction] - auto dp_frac = get_field_in("dp_frac").get_view(); auto icwmrsh = get_field_out("icwmrsh") .get_view(); // in cloud water mixing ratio, @@ -545,8 +537,6 @@ void MAMWetscav::run_impl(const double dt) { mam4::Tendencies tends = mam_coupling::interstitial_aerosols_tendencies_for_column( dry_aero_tends, icol); - // shallow_convective_cloud_fraction - auto cldn_icol = ekat::subview(cldn, icol); /// shallow_convective_precipitation_production auto rprdsh_icol = ekat::subview(rprdsh, icol); // deep_convective_precipitation_production @@ -581,9 +571,9 @@ void MAMWetscav::run_impl(const double dt) { mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, // inputs - cldt_icol, cldn_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, - evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, - evapr_icol, dlf_icol, prain_icol, + cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, + dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, + dlf_icol, prain_icol, // in/out wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, // output diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index f1e0194106a2..0096ea4cc635 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -25,14 +25,10 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - cldn_prev_step: 0.5 - cldn: 0.5 rprdsh: 1e-5 rprddp: 1e-5 evapcsh: 1e-5 evapcdp: 1e-5 - cldt_prev_step: 1e-5 - cldt: 1e-5 cldst: 1e-5 qme: 1e-5 prain: 1e-5 @@ -41,8 +37,8 @@ initial_conditions: icwmrsh: 1e-5 rprddp: 1e-5 rprdsh: 1e-5 - sh_frac: 1e-5 - dp_frac: 1e-5 + sh_frac: 0. + dp_frac: 0. evapcsh: 1e-5 evapcdp: 1e-5 bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] From 6ada930cc3355152519342f26f48372c8f0a8aff Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 20 Jun 2024 21:23:10 -0700 Subject: [PATCH 101/477] Makes some more convective fields intent-in (updates submodule mam4xx) --- .../eamxx_mam_wetscav_process_interface.cpp | 77 +++++++++---------- .../single-process/mam/wet_scav/input.yaml | 12 +-- 2 files changed, 39 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index e71bda90ca94..817b8eeaed7e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -80,8 +80,7 @@ void MAMWetscav::set_grids( add_field("p_int", scalar3d_int, Pa, grid_name); // total pressure - // -- Input variables that exists in PBUF in EAM - // for variables that are fractions etc. + // For variables that are non dimensional (e.g., fractions etc.) static constexpr auto nondim = Units::nondimensional(); // Deep convective cloud fraction [fraction] @@ -93,18 +92,21 @@ void MAMWetscav::set_grids( // Total cloud fraction [fraction] add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); - add_field("rprdsh", scalar3d_mid, kg / kg / s, - grid_name); // rain production, shallow convection - // [kg/kg/s] //NOT updated - add_field( - "rprddp", scalar3d_mid, kg / kg / s, - grid_name); // rain production, deep convection [kg/kg/s] //NOT updated - add_field("evapcsh", scalar3d_mid, kg / kg / s, - grid_name); // Evaporation rate of shallow convective - // //NOT updated precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_mid, kg / kg / s, - grid_name); // Evaporation rate of deep convective //NOT - // updated precipitation >=0. [kg/kg/s] + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); + + // Evaporation rate of deep precipitation >=0. [kg/kg/s] + add_field("evapcdp", scalar3d_mid, kg / kg / s, grid_name); + + // Rain production, shallow convection [kg/kg/s] + add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); + + // Rain production, deep convection [kg/kg/s] + add_field("rprddp", scalar3d_mid, kg / kg / s, grid_name); + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // ------------------------------------------------------------------------------------------------------------------------- // -- Input variables that exists in PBUF in EAM (in wetdep.F90) in the // "inputs" data structure @@ -126,22 +128,12 @@ void MAMWetscav::set_grids( add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); // In cloud water mixing ratio, shallow // convection [kg/kg] //NOT updated - // add_field("rprddp", scalar3d_mid, kg / kg / s, - // grid_name); // Rain production, deep convection - // [kg/kg/s] // in cloud water mixing ratio, deep shallow [kg/kg] add_field("icwmrsh", scalar3d_mid, nondim, grid_name); // add_field( // "icwmrdp", scalar3d_mid, nondim, // grid_name); - // add_field("evapcsh", scalar3d_mid, kg / kg / s, - // grid_name); // Evaporation rate of shallow convective - // // precipitation >=0. [kg/kg/s] - // add_field("evapcdp", scalar3d_mid, kg / kg / s, - // grid_name); // Evaporation rate of deep convective - // // precipitation >=0. [kg/kg/s] - // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -466,7 +458,9 @@ void MAMWetscav::run_impl(const double dt) { const auto &work = work_; const auto &dry_aero_tends = dry_aero_tends_; - // inputs + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- // Shallow convective cloud fraction [fraction] auto sh_frac = get_field_in("sh_frac").get_view(); @@ -474,26 +468,25 @@ void MAMWetscav::run_impl(const double dt) { auto dp_frac = get_field_in("dp_frac").get_view(); // Total cloud fraction - auto cldt = get_field_in("cldfrac_liq").get_view(); //?? + auto cldt = get_field_in("cldfrac_liq").get_view(); + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + auto evapcsh = get_field_in("evapcsh").get_view(); + + // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] + auto evapcdp = get_field_in("evapcdp").get_view(); - // inputs/outputs + // Rain production, shallow convection [kg/kg/s] + auto rprdsh = get_field_in("rprdsh").get_view(); + + // Rain production, deep convection [kg/kg/s] + auto rprddp = get_field_in("rprddp").get_view(); + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "Updated" or pure inputs/outputs for the process + // ------------------------------------------------------------------------------------------------------------------------- auto dlf = get_field_out("dlf").get_view(); auto evapr = get_field_out("evapr").get_view(); - auto rprdsh = - get_field_out("rprdsh").get_view(); // rain production, shallow - - // convection [kg/kg/s] - auto evapcsh = - get_field_out("evapcsh") - .get_view(); // Evaporation rate of shallow convective - // precipitation >=0. [kg/kg/s] - auto rprddp = - get_field_out("rprddp") - .get_view(); // rain production, deep convection [kg/kg/s] - auto evapcdp = - get_field_out("evapcdp") - .get_view(); // Evaporation rate of deep convective - // precipitation >=0. [kg/kg/s] auto icwmrsh = get_field_out("icwmrsh") .get_view(); // in cloud water mixing ratio, diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 0096ea4cc635..46bf2db5740b 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -25,22 +25,18 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - rprdsh: 1e-5 - rprddp: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 + rprdsh: 0. + rprddp: 0. + evapcsh: 0. + evapcdp: 0. cldst: 1e-5 qme: 1e-5 prain: 1e-5 evapr: 1e-5 icwmrdp: 1e-5 icwmrsh: 1e-5 - rprddp: 1e-5 - rprdsh: 1e-5 sh_frac: 0. dp_frac: 0. - evapcsh: 1e-5 - evapcdp: 1e-5 bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] From 04ae76410354e6b4d7a5c86ebacbdac6f99ace01 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 21 Jun 2024 07:01:01 -0700 Subject: [PATCH 102/477] Fixes some more vars to be intent-in (mam4xx submodule updated) --- .../eamxx_mam_wetscav_process_interface.cpp | 46 +++++++++---------- .../single-process/mam/wet_scav/input.yaml | 4 +- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 817b8eeaed7e..ac8a1fae4419 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -104,6 +104,16 @@ void MAMWetscav::set_grids( // Rain production, deep convection [kg/kg/s] add_field("rprddp", scalar3d_mid, kg / kg / s, grid_name); + // In cloud water mixing ratio, deep convection [kg/kg] + add_field("icwmrdp", scalar3d_mid, kg / kg, grid_name); + + // In cloud water mixing ratio, shallow convection [kg/kg] + add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); + + // evaporation from stratiform rain [kg/kg/s] + // FIXME: Get it from P3 + add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -117,23 +127,9 @@ void MAMWetscav::set_grids( // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // stratiform rain production rate [kg/kg/s] - add_field( - "evapr", scalar3d_mid, kg / kg / s, - grid_name); // evaporation from stratiform rain [kg/kg/s] //NOT updated // -- Input variables that exists in PBUF in EAM (in wetdep.F90) - add_field("icwmrdp", scalar3d_mid, kg / kg, - grid_name); // In cloud water mixing ratio, deep - // convection [kg/kg] //NOT updated - add_field("icwmrsh", scalar3d_mid, kg / kg, - grid_name); // In cloud water mixing ratio, shallow - // convection [kg/kg] //NOT updated - - // in cloud water mixing ratio, deep shallow [kg/kg] - add_field("icwmrsh", scalar3d_mid, nondim, grid_name); - // add_field( - // "icwmrdp", scalar3d_mid, nondim, - // grid_name); + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -482,18 +478,20 @@ void MAMWetscav::run_impl(const double dt) { // Rain production, deep convection [kg/kg/s] auto rprddp = get_field_in("rprddp").get_view(); + // In cloud water mixing ratio, deep convection + auto icwmrdp = get_field_in("icwmrdp").get_view(); + + // In cloud water mixing ratio, shallow convection + auto icwmrsh = get_field_in("icwmrsh").get_view(); + + // evaporation from stratiform rain [kg/kg/s] + // FIXME: Get it from P3 + auto evapr = get_field_in("evapr").get_view(); + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "Updated" or pure inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- - auto dlf = get_field_out("dlf").get_view(); - auto evapr = get_field_out("evapr").get_view(); - - auto icwmrsh = get_field_out("icwmrsh") - .get_view(); // in cloud water mixing ratio, - // shallow convection - auto icwmrdp = - get_field_out("icwmrdp") - .get_view(); // in cloud water mixing ratio, deep convection + auto dlf = get_field_out("dlf").get_view(); auto prain = get_field_out("prain") .get_view(); // stratiform rain production rate diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 46bf2db5740b..13447577cf8f 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -33,8 +33,8 @@ initial_conditions: qme: 1e-5 prain: 1e-5 evapr: 1e-5 - icwmrdp: 1e-5 - icwmrsh: 1e-5 + icwmrdp: 0. + icwmrsh: 0. sh_frac: 0. dp_frac: 0. bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] From b4ffbe9b0ce30a7d1779c472506a26939f37643a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 12:53:07 -0700 Subject: [PATCH 103/477] Moved around variables to match I/O-1 --- .../eamxx_mam_wetscav_process_interface.cpp | 181 ++++++++++-------- 1 file changed, 97 insertions(+), 84 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index ac8a1fae4419..b003cb22e855 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -64,34 +64,72 @@ void MAMWetscav::set_grids( // layout for 2D (ncol, pcnst) FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "pcnst"); - // ------------------------------------------------------------------------------------------------------------------------- + + // -------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_mid, Pa, - grid_name); // pressure at mid points in [Pa - add_field("pseudo_density", scalar3d_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qc", scalar3d_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("p_int", scalar3d_int, Pa, - grid_name); // total pressure + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg] + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + // planetary boundary layer height + add_field("pbl_height", scalar2d, m, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] (FIXME: Get it from P3) + add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); + + // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) + add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // For variables that are non dimensional (e.g., fractions etc.) static constexpr auto nondim = Units::nondimensional(); + //----------- Variables from macrophysics scheme ------------- + + // Total cloud fraction [fraction] + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); + + //----------- Variables from convective scheme ------------- + + // Following variables are from convective parameterization (not implemented + // yet in EAMxx), so should be zero for now + // Deep convective cloud fraction [fraction] add_field("dp_frac", scalar3d_mid, nondim, grid_name); // Shallow convective cloud fraction [fraction] //NOT updated add_field("sh_frac", scalar3d_mid, nondim, grid_name); - // Total cloud fraction [fraction] - add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); - // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); @@ -110,26 +148,6 @@ void MAMWetscav::set_grids( // In cloud water mixing ratio, shallow convection [kg/kg] add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); - // evaporation from stratiform rain [kg/kg/s] - // FIXME: Get it from P3 - add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); - - // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "updated" or inputs/outputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - - // -- Input variables that exists in PBUF in EAM (in wetdep.F90) in the - // "inputs" data structure - - // FIXME: we do not need qme - // add_field( - // "qme", scalar3d_mid, kg / kg / s, - // grid_name); // net condensation/evaporation of cloud water [kg/kg/s] - add_field("prain", scalar3d_mid, kg / kg / s, - grid_name); // stratiform rain production rate [kg/kg/s] - - // -- Input variables that exists in PBUF in EAM (in wetdep.F90) - // ------------------------------------------------------------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -137,48 +155,37 @@ void MAMWetscav::set_grids( // -- surface fluxes (input/outpts) for the coupler's cam_out data struture // for the land model static constexpr auto m2 = m * m; - add_field( - "bcphiwet", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of hydrophilic black carbon [kg/m2/s] - add_field( - "bcphidry", scalar3d_mid, kg / m2 / s, - grid_name); // dry deposition of hydrophilic black carbon [kg/m2/s] - add_field( - "ocphiwet", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of hydrophilic organic carbon [kg/m2/s] - add_field( - "ocphidry", scalar3d_mid, kg / m2 / s, - grid_name); // dry deposition of hydrophilic organic carbon [kg/m2/s] - - add_field("dstwet1", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of dust (bin1) [kg/m2/s] - add_field("dstwet2", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of dust (bin2) [kg/m2/s] - add_field("dstwet3", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of dust (bin3) [kg/m2/s] - add_field("dstwet4", scalar3d_mid, kg / m2 / s, - grid_name); // wet deposition of dust (bin4) [kg/m2/s] - - // -- input/ouputs from PBUF for updating particle size and water uptake by - // particles - static constexpr auto m3 = m2 * m; - add_field("dgncur_a", scalar3d_mid_nmodes, m, - grid_name); // aerosol dry particle diameter [m] - add_field("wetdens", scalar3d_mid_nmodes, kg / m3, - grid_name); // wet aerosol density [kg/m3] - add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, - grid_name); // aerosol water [kg/kg] - // - add_field("dgnumwet", scalar3d_mid_nmodes, m, - grid_name); // wet aerosol diameter [m] - add_field("fracis", scalar3d_mid, nondim, - grid_name); // fraction of transported species that are - // insoluble [fraction] - // -- interstitial and cloudborne aerosol tracers of interest: mass (q) and + // Wet deposition of hydrophilic black carbon [kg/m2/s] + add_field("bcphiwet", scalar3d_mid, kg / m2 / s, grid_name); + + // Dry deposition of hydrophilic black carbon [kg/m2/s] + add_field("bcphidry", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of hydrophilic organic carbon [kg/m2/s] + add_field("ocphiwet", scalar3d_mid, kg / m2 / s, grid_name); + + // Dry deposition of hydrophilic organic carbon [kg/m2/s] + add_field("ocphidry", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin1) [kg/m2/s] + add_field("dstwet1", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin2) [kg/m2/s] + add_field("dstwet2", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin3) [kg/m2/s] + add_field("dstwet3", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin4) [kg/m2/s] + add_field("dstwet4", scalar3d_mid, kg / m2 / s, grid_name); + + // Interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios - // -- NOTE: Interstitial aerosols are updated in the interface using the + + // NOTE: Interstitial aerosols are updated in the interface using the // "tendencies" from the wetscavenging process + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = @@ -228,6 +235,20 @@ void MAMWetscav::set_grids( "tracers"); } + static constexpr auto m3 = m2 * m; + add_field("dgncur_a", scalar3d_mid_nmodes, m, + grid_name); // aerosol dry particle diameter [m] + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, + grid_name); // wet aerosol density [kg/m3] + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, + grid_name); // aerosol water [kg/kg] + // + add_field("dgnumwet", scalar3d_mid_nmodes, m, + grid_name); // wet aerosol diameter [m] + add_field("fracis", scalar3d_mid, nondim, + grid_name); // fraction of transported species that are + // insoluble [fraction] + // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { std::string ptend_gas_name = @@ -261,14 +282,6 @@ void MAMWetscav::set_grids( static constexpr auto s2 = s * s; add_field("phis", scalar2d, m2 / s2, grid_name); // surface geopotential - add_field("qv", scalar3d_mid, q_unit, - grid_name); // specific humidity - add_field("nc", scalar3d_mid, n_unit, - grid_name); // cloud water number conc - add_field("ni", scalar3d_mid, n_unit, - grid_name); // cloud ice number conc - add_field("omega", scalar3d_mid, Pa / s, - grid_name); // vertical pressure velocity // FIXME: units add_field("dlf", scalar3d_mid, kg / kg / s, From abd75db1ca56d4a1ebc71ebf0365be03521e4a83 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 19:27:55 -0700 Subject: [PATCH 104/477] Removes ptend from output FM and some cleanup for I/O-2 --- .../eamxx_mam_wetscav_process_interface.cpp | 94 ++++++------------- .../single-process/mam/wet_scav/output.yaml | 25 ----- 2 files changed, 28 insertions(+), 91 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index b003cb22e855..148353e793ea 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -148,9 +148,9 @@ void MAMWetscav::set_grids( // In cloud water mixing ratio, shallow convection [kg/kg] add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); - // ------------------------------------------------------------------------------------------------------------------------- + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process - // ------------------------------------------------------------------------------------------------------------------------- + // --------------------------------------------------------------------- // FIXME: we have not added code to update the surface fluxes. // -- surface fluxes (input/outpts) for the coupler's cam_out data struture // for the land model @@ -235,47 +235,26 @@ void MAMWetscav::set_grids( "tracers"); } + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- static constexpr auto m3 = m2 * m; - add_field("dgncur_a", scalar3d_mid_nmodes, m, - grid_name); // aerosol dry particle diameter [m] - add_field("wetdens", scalar3d_mid_nmodes, kg / m3, - grid_name); // wet aerosol density [kg/m3] - add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, - grid_name); // aerosol water [kg/kg] - // - add_field("dgnumwet", scalar3d_mid_nmodes, m, - grid_name); // wet aerosol diameter [m] - add_field("fracis", scalar3d_mid, nondim, - grid_name); // fraction of transported species that are - // insoluble [fraction] - // aerosol-related gases: mass mixing ratios - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_gas_name = - "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); - add_field(ptend_gas_name, scalar3d_mid, dqdt_unit, grid_name); - } + // Aerosol dry particle diameter [m] + add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); - // tendencies for interstitial aerosols - for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_num = - "ptend_" + std::string(mam_coupling::int_aero_nmr_field_name(imode)); - add_field(ptend_num, scalar3d_mid, n_unit, grid_name); - for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(imode, ispec); - if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_mmr_field_name = - "ptend_" + std::string(int_mmr_field_name); - add_field(ptend_int_mmr_field_name, scalar3d_mid, dqdt_unit, - grid_name); - } - } - } + // wet aerosol density [kg/m3] + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); + + // aerosol water [kg/kg] + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); + + // wet aerosol diameter [m] + add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); + + // fraction of transported species that are insoluble [fraction] + add_field("fracis", scalar3d_mid, nondim, grid_name); - add_field("cldfrac_tot", scalar3d_mid, nondim, - grid_name); // Cloud fraction add_field("pbl_height", scalar2d, m, grid_name); // PBL height @@ -283,13 +262,9 @@ void MAMWetscav::set_grids( add_field("phis", scalar2d, m2 / s2, grid_name); // surface geopotential - // FIXME: units - add_field("dlf", scalar3d_mid, kg / kg / s, - grid_name); // - // add_field("dp_ccf", scalar3d_mid, n_unit, - // grid_name); // - // add_field("sh_ccf", scalar3d_mid, n_unit, - // grid_name); // + add_field("dlf", scalar3d_mid, kg / kg / s, + grid_name); // + // aerosol wet deposition (interstitial) add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); // @@ -366,7 +341,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { // The following dry_atm_ members *may* not be used by the process but they // are needed for creating MAM4xx class objects like Atmosphere - dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_liq").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.phis = get_field_in("phis").get_view(); dry_atm_.z_surf = 0.0; @@ -412,30 +387,17 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } - // ---- set aerosol-related gas tendencies data + // Alllocate aerosol-related gas tendencies for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - std::string ptend_mmr_field_name = - "ptend_" + std::string(mam_coupling::gas_mmr_field_name(g)); - dry_aero_tends_.gas_mmr[g] = - get_field_out(ptend_mmr_field_name).get_view(); + Kokkos::resize(dry_aero_tends_.gas_mmr[g], ncol_, nlev_); } - // set aerosol state tendencies data (interstitial aerosols only) + // Allocate aerosol state tendencies (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - std::string ptend_int_nmr_field_name = - "ptend_" + std::string(mam_coupling::int_aero_nmr_field_name(imode)); - dry_aero_tends_.int_aero_nmr[imode] = - get_field_out(ptend_int_nmr_field_name).get_view(); + Kokkos::resize(dry_aero_tends_.int_aero_nmr[imode], ncol_, nlev_); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(imode, ispec); - if(strlen(int_mmr_field_name) > 0) { - std::string ptend_int_aero_mmr_field_name = - "ptend_" + std::string(int_mmr_field_name); - dry_aero_tends_.int_aero_mmr[imode][ispec] = - get_field_out(ptend_int_aero_mmr_field_name).get_view(); - } + Kokkos::resize(dry_aero_tends_.int_aero_mmr[imode][ispec], ncol_, nlev_); } } @@ -504,7 +466,7 @@ void MAMWetscav::run_impl(const double dt) { // ------------------------------------------------------------------------------------------------------------------------- // These variables are "Updated" or pure inputs/outputs for the process // ------------------------------------------------------------------------------------------------------------------------- - auto dlf = get_field_out("dlf").get_view(); + auto dlf = get_field_in("dlf").get_view(); auto prain = get_field_out("prain") .get_view(); // stratiform rain production rate diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index b9f8bc107fc1..9cd0a16bc414 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -11,31 +11,6 @@ Fields: - dgncur_a - wetdens - qaerwat - - ptend_bc_a1 - - ptend_bc_a3 - - ptend_bc_a4 - - ptend_dst_a1 - - ptend_dst_a3 - - ptend_mom_a1 - - ptend_mom_a2 - - ptend_mom_a3 - - ptend_mom_a4 - - ptend_nacl_a1 - - ptend_nacl_a2 - - ptend_nacl_a3 - - ptend_num_a1 - - ptend_num_a2 - - ptend_num_a3 - - ptend_num_a4 - - ptend_pom_a1 - - ptend_pom_a3 - - ptend_pom_a4 - - ptend_so4_a1 - - ptend_so4_a2 - - ptend_so4_a3 - - ptend_soa_a1 - - ptend_soa_a2 - - ptend_soa_a3 - bc_c1 - bc_c3 - bc_c4 From 6c819c8aba0687b8f9ec9a848448af360bc70b80 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 19:43:06 -0700 Subject: [PATCH 105/477] Done with the initialize_impl method cleanup --- .../eamxx_mam_wetscav_process_interface.cpp | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 148353e793ea..daa203ab7b88 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -100,9 +100,15 @@ void MAMWetscav::set_grids( // Layer thickness(pdel) [Pa] at midpoints add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - // planetary boundary layer height + // planetary boundary layer height [m] add_field("pbl_height", scalar2d, m, grid_name); + static constexpr auto m2 = m * m; + static constexpr auto s2 = s * s; + + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); + //----------- Variables from microphysics scheme ------------- // Evaporation from stratiform rain [kg/kg/s] (FIXME: Get it from P3) @@ -148,13 +154,15 @@ void MAMWetscav::set_grids( // In cloud water mixing ratio, shallow convection [kg/kg] add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); + // Detraining cld H20 from deep convection [kg/ks/s] + add_field("dlf", scalar3d_mid, kg / kg / s, grid_name); + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- // FIXME: we have not added code to update the surface fluxes. // -- surface fluxes (input/outpts) for the coupler's cam_out data struture // for the land model - static constexpr auto m2 = m * m; // Wet deposition of hydrophilic black carbon [kg/m2/s] add_field("bcphiwet", scalar3d_mid, kg / m2 / s, grid_name); @@ -255,22 +263,10 @@ void MAMWetscav::set_grids( // fraction of transported species that are insoluble [fraction] add_field("fracis", scalar3d_mid, nondim, grid_name); - add_field("pbl_height", scalar2d, m, - grid_name); // PBL height - - static constexpr auto s2 = s * s; - add_field("phis", scalar2d, m2 / s2, - grid_name); // surface geopotential - - add_field("dlf", scalar3d_mid, kg / kg / s, - grid_name); // - - // aerosol wet deposition (interstitial) - add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, - grid_name); // - // aerosol wet deposition (cloud water) - add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, - grid_name); // + // aerosol wet deposition (interstitial) [kg/m2/s] + add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); + // aerosol wet deposition (cloud water) [kg/m2/s] + add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); } // ========================================================================================= From 3e73b8ffc0fa82c734421f4a42dbbc9ed5e1a001 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 2 Jul 2024 22:38:54 -0700 Subject: [PATCH 106/477] Rebased on current master --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 5 ++--- externals/mam4xx | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index daa203ab7b88..ef7661c2ba85 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -297,7 +297,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); // populate the wet atmosphere state with views from fields and - // the buffer (NOTE: wet atmosphere only has qv, qc, qi, nc, ni and omega) + // the buffer (NOTE: wet atmosphere only has qv, qc, qi, nc, ni) wet_atm_.qc = get_field_in("qc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); @@ -307,8 +307,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { wet_atm_.qv = get_field_in("qv").get_view(); wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); - // populate the dry atmosphere state with views from fields // (NOTE: dry atmosphere has everything that wet // atmosphere has along with z_surf, T_mid, p_mid, z_mid, z_iface, @@ -317,6 +315,7 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); // How "buffer_" works: We use buffer to allocate memory for the members of // dry_atm_ object. Here we are providing those memory locations to the // dry_atm_ members. These members are computed from the above wet_atm_ or diff --git a/externals/mam4xx b/externals/mam4xx index a8aa56ba915c..1b5f3b6f4cd9 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e +Subproject commit 1b5f3b6f4cd90b18869e20acc4e68663b3c4e0e8 From f52f6284305ae10a87ad9c2d46d686a9a9811dae Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Jul 2024 06:47:26 -0700 Subject: [PATCH 107/477] Completed a round of cleanup- variables rearranged to match i/o-3 --- .../eamxx_mam_wetscav_process_interface.cpp | 114 ++++++++++-------- .../single-process/mam/wet_scav/input.yaml | 23 ++-- externals/mam4xx | 2 +- 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index ef7661c2ba85..713078d76623 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -115,7 +115,7 @@ void MAMWetscav::set_grids( add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) - add_field("prain", scalar3d_mid, kg / kg / s, grid_name); + add_field("prain", scalar3d_mid, kg / kg / s, grid_name); // For variables that are non dimensional (e.g., fractions etc.) static constexpr auto nondim = Units::nondimensional(); @@ -154,7 +154,7 @@ void MAMWetscav::set_grids( // In cloud water mixing ratio, shallow convection [kg/kg] add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); - // Detraining cld H20 from deep convection [kg/ks/s] + // Detraining cld H20 from deep convection [kg/kg/s] add_field("dlf", scalar3d_mid, kg / kg / s, grid_name); // --------------------------------------------------------------------- @@ -293,37 +293,38 @@ void MAMWetscav::init_buffers(const ATMBufferManager &buffer_manager) { // ========================================================================================= void MAMWetscav::initialize_impl(const RunType run_type) { - // Gather runtime options - //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- - // populate the wet atmosphere state with views from fields and - // the buffer (NOTE: wet atmosphere only has qv, qc, qi, nc, ni) + // store fields only to be converted to dry mmrs in wet_atm_ wet_atm_.qc = get_field_in("qc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); - // -- Following wet atm variables are NOT used by the process but we still - // need them to - // -- create atmosphere object - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - // populate the dry atmosphere state with views from fields + // Following wet atm variables are NOT used by the process but we still + // need them to create atmosphere object + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields // (NOTE: dry atmosphere has everything that wet // atmosphere has along with z_surf, T_mid, p_mid, z_mid, z_iface, - // dz, p_del, cldfrac, w_updraft, pblh, phis) - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); + // dz, p_del, cldfrac, w_updraft, pblh, phis and omega) + dry_atm_.z_surf = 0; + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_liq").get_view(); + + // The following dry_atm_ members *may* not be used by the process but they + // are needed for creating MAM4xx class objects like Atmosphere dry_atm_.omega = get_field_in("omega").get_view(); - // How "buffer_" works: We use buffer to allocate memory for the members of - // dry_atm_ object. Here we are providing those memory locations to the - // dry_atm_ members. These members are computed from the above wet_atm_ or - // dry_atm_ members that are explicitly getting their values either from the - // input file or from other processes. These members are null at this point, - // they are assigned in "Kokkos::parallel_for("preprocess", scan_policy, - // preprocess_);" call in the run_impl + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.qv = buffer_.qv_dry; dry_atm_.qc = buffer_.qc_dry; dry_atm_.nc = buffer_.nc_dry; @@ -334,12 +335,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_atm_.z_iface = buffer_.z_iface; dry_atm_.w_updraft = buffer_.w_updraft; - // The following dry_atm_ members *may* not be used by the process but they - // are needed for creating MAM4xx class objects like Atmosphere - dry_atm_.cldfrac = get_field_in("cldfrac_liq").get_view(); - dry_atm_.pblh = get_field_in("pbl_height").get_view(); - dry_atm_.phis = get_field_in("phis").get_view(); - dry_atm_.z_surf = 0.0; // ---- set wet/dry aerosol-related gas state data for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); @@ -382,6 +377,10 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } + //--------------------------------------------------------------------------------- + // Allocate memory + // (Kokkos::resize only works on host to allocates memory) + //--------------------------------------------------------------------------------- // Alllocate aerosol-related gas tendencies for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { Kokkos::resize(dry_aero_tends_.gas_mmr[g], ncol_, nlev_); @@ -396,29 +395,31 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } - // set up our preprocess/postprocess functors - // Here we initialize (not compute) objects in preprocess struct using the - // objects in the argument list + // Allocate work array + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); + work_ = view_2d("work", ncol_, work_len); + + //--------------------------------------------------------------------------------- + // Setup preprocessing and post processing + //--------------------------------------------------------------------------------- + // set up our preprocess and postprocess functors preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - - // wetdep - - const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); - work_ = view_2d("work", ncol_, work_len); } // ========================================================================================= void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // preprocess input -- needs a scan for the calculation of all variables // needed by this process or setting up MAM4xx classes and their objects Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; const auto &dry_aero = dry_aero_; const auto &work = work_; @@ -427,15 +428,17 @@ void MAMWetscav::run_impl(const double dt) { // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - // Shallow convective cloud fraction [fraction] + + //----------- Variables from convective scheme ------------- + + // Following variables are from convective parameterization (not implemented + // yet in EAMxx), so should be zero for now + auto sh_frac = get_field_in("sh_frac").get_view(); // Deep convective cloud fraction [fraction] auto dp_frac = get_field_in("dp_frac").get_view(); - // Total cloud fraction - auto cldt = get_field_in("cldfrac_liq").get_view(); - // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] auto evapcsh = get_field_in("evapcsh").get_view(); @@ -454,18 +457,24 @@ void MAMWetscav::run_impl(const double dt) { // In cloud water mixing ratio, shallow convection auto icwmrsh = get_field_in("icwmrsh").get_view(); - // evaporation from stratiform rain [kg/kg/s] - // FIXME: Get it from P3 + // Detraining cld H20 from deep convection [kg/kg/s] + auto dlf = get_field_in("dlf").get_view(); + + //----------- Variables from macrophysics scheme ------------- + // Total cloud fraction + auto cldt = get_field_in("cldfrac_liq").get_view(); + + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] auto evapr = get_field_in("evapr").get_view(); + // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) + auto prain = get_field_in("prain").get_view(); // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "Updated" or pure inputs/outputs for the process + // These variables are "Computed" or pure outputs for the process // ------------------------------------------------------------------------------------------------------------------------- - auto dlf = get_field_in("dlf").get_view(); - auto prain = get_field_out("prain") - .get_view(); // stratiform rain production rate - // outputs const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); @@ -483,7 +492,7 @@ void MAMWetscav::run_impl(const double dt) { // inside a parallel_for. const int nlev = nlev_; - // loop over atmosphere columns and compute aerosol particle size + // Loop over atmosphere columns Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { const int icol = team.league_rank(); // column index*/ @@ -535,9 +544,8 @@ void MAMWetscav::run_impl(const double dt) { cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, dlf_icol, prain_icol, - // in/out + // outputs wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, - // output aerdepwetis_icol, aerdepwetcw_icol, work_icol); team.team_barrier(); // update interstitial aerosol state diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index 13447577cf8f..c481165d29ad 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -25,18 +25,22 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables from P3 + prain: 1e-5 + evapr: 1e-5 + #variables from convection scheme (they should all be zero) rprdsh: 0. rprddp: 0. evapcsh: 0. evapcdp: 0. - cldst: 1e-5 - qme: 1e-5 - prain: 1e-5 - evapr: 1e-5 icwmrdp: 0. icwmrsh: 0. sh_frac: 0. dp_frac: 0. + dlf : 0. + #surface fluxes bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] @@ -45,17 +49,6 @@ initial_conditions: dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] - dgncur_a: 1e-5 # aerosol particle diameter [m] - wetdens: 1e-5 # wet aerosol density [kg/m3] - qaerwat: 1e-5 # aerosol water [kg/kg] - dgnumwet: 1e-5 # wet aerosol diameter [m] - fracis: 1e-5 # fraction of transported species that are insoluble [fraction] - cldfrac_tot: 0.5 - pbl_height: 25.0 - phis : 0.1 - dlf : 0.0 - dp_ccf : 0.0 - sh_ccf : 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] diff --git a/externals/mam4xx b/externals/mam4xx index 1b5f3b6f4cd9..f5700d5f2ac6 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 1b5f3b6f4cd90b18869e20acc4e68663b3c4e0e8 +Subproject commit f5700d5f2ac6a6a7f845ee73a50a2bd1d268e22c From 870e9785261f6b26d4c0b4c253213e0ca1981628 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Jul 2024 09:41:56 -0700 Subject: [PATCH 108/477] Adds a multi-process test: p3-wetscav --- .../eamxx_mam_wetscav_process_interface.cpp | 24 ++++++++++--------- .../src/share/io/scream_scorpio_interface.cpp | 4 ++-- .../multi-process/physics_only/CMakeLists.txt | 1 + .../single-process/mam/wet_scav/input.yaml | 4 ++-- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 713078d76623..c581a5757064 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -10,9 +10,10 @@ //>>>>>>> /* -Future work: -Wirte comments -write in/outs for all variables clearly +----------------------------------------------------------------- +NOTES: +1. We should connect surface fluxes and add code to update the fluxes +2. Identify diagnostic variables and remove them from FM */ namespace scream { @@ -112,10 +113,11 @@ void MAMWetscav::set_grids( //----------- Variables from microphysics scheme ------------- // Evaporation from stratiform rain [kg/kg/s] (FIXME: Get it from P3) - add_field("evapr", scalar3d_mid, kg / kg / s, grid_name); + add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) - add_field("prain", scalar3d_mid, kg / kg / s, grid_name); + add_field("precip_total_tend", scalar3d_mid, kg / kg / s, + grid_name); // For variables that are non dimensional (e.g., fractions etc.) static constexpr auto nondim = Units::nondimensional(); @@ -425,9 +427,9 @@ void MAMWetscav::run_impl(const double dt) { const auto &work = work_; const auto &dry_aero_tends = dry_aero_tends_; - // ------------------------------------------------------------------------------------------------------------------------- + // --------------------------------------------------------------- // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- + // --------------------------------------------------------------- //----------- Variables from convective scheme ------------- @@ -467,10 +469,10 @@ void MAMWetscav::run_impl(const double dt) { //----------- Variables from microphysics scheme ------------- // Evaporation from stratiform rain [kg/kg/s] - auto evapr = get_field_in("evapr").get_view(); + auto nevapr = get_field_in("nevapr").get_view(); // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) - auto prain = get_field_in("prain").get_view(); + auto prain = get_field_in("precip_total_tend").get_view(); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "Computed" or pure outputs for the process // ------------------------------------------------------------------------------------------------------------------------- @@ -523,7 +525,7 @@ void MAMWetscav::run_impl(const double dt) { auto icwmrdp_col = ekat::subview(icwmrdp, icol); auto icwmrsh_icol = ekat::subview(icwmrsh, icol); - auto evapr_icol = ekat::subview(evapr, icol); + auto nevapr_icol = ekat::subview(nevapr, icol); auto cldt_icol = ekat::subview(cldt, icol); auto dlf_icol = ekat::subview(dlf, icol); @@ -542,7 +544,7 @@ void MAMWetscav::run_impl(const double dt) { team, atm, progs, tends, dt, // inputs cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, - dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, evapr_icol, + dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, nevapr_icol, dlf_icol, prain_icol, // outputs wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index cb2b501f2b5c..68a432819ec3 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - for (auto& it : s.files) { + /*for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - } + }*/ s.files.clear(); for (auto& it : s.decomps) { diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index b9c13234796b..36b43c47c173 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -8,6 +8,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) + add_subdirectory(mam/p3_mam4_wetscav) endif() endif() diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index c481165d29ad..f2a62a15823e 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -28,8 +28,8 @@ initial_conditions: pbl_height: 25.0 phis : 0.1 #variables from P3 - prain: 1e-5 - evapr: 1e-5 + precip_total_tend: 1e-5 + nevapr: 1e-5 #variables from convection scheme (they should all be zero) rprdsh: 0. rprddp: 0. From 8a81e1c18bf670523cc4908a6d0e5a4daaeae359 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Jul 2024 09:43:17 -0700 Subject: [PATCH 109/477] Adds missing files for the new multi-process test --- .../mam/p3_mam4_wetscav/CMakeLists.txt | 41 ++++++++++++ .../mam/p3_mam4_wetscav/input.yaml | 67 +++++++++++++++++++ .../mam/p3_mam4_wetscav/output.yaml | 66 ++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..7e13932e0632 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME p3_mam4_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS p3 mam + LABELS p3 physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep p3 dt<=300s +set (P3_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${P3_MAX_DT} - 1) / ${P3_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS p3 physics mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml new file mode 100644 index 000000000000..c2c422ca2e94 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml @@ -0,0 +1,67 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic, mam4_wetscav] + schedule_type: Sequential + mac_mic: + atm_procs_list: [p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables needed by mam4_wetscav + #variables from convection scheme (they should all be zero) + rprdsh: 0. + rprddp: 0. + evapcsh: 0. + evapcdp: 0. + icwmrdp: 0. + icwmrsh: 0. + sh_frac: 0. + dp_frac: 0. + dlf : 0. + #surface fluxes + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + + #variable required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml new file mode 100644 index 000000000000..d8d8e7694709 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -0,0 +1,66 @@ +%YAML 1.1 +--- +filename_prefix: shoc_mam4_aci_output +Averaging Type: Instant +Field Names: + - cldfrac_liq + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 64fc5a3c82945c1209833f3d9c3d9c990bacbab0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Jul 2024 10:59:30 -0700 Subject: [PATCH 110/477] Adds another multi-process test for wetscav --- .../multi-process/physics_only/CMakeLists.txt | 1 + .../mam/p3_mam4_wetscav/input.yaml | 4 +- .../mam/p3_mam4_wetscav/output.yaml | 2 +- .../shoc_cldfrac_p3_wetscav/CMakeLists.txt | 41 +++++++ .../mam/shoc_cldfrac_p3_wetscav/input.yaml | 83 +++++++++++++++ .../mam/shoc_cldfrac_p3_wetscav/output.yaml | 100 ++++++++++++++++++ 6 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 36b43c47c173..d3b584bd09e4 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -9,6 +9,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) add_subdirectory(mam/p3_mam4_wetscav) + add_subdirectory(mam/shoc_cldfrac_p3_wetscav) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml index c2c422ca2e94..f5d2dc036c0b 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml @@ -37,7 +37,7 @@ initial_conditions: pbl_height: 25.0 phis : 0.1 #variables needed by mam4_wetscav - #variables from convection scheme (they should all be zero) + #--variables from convection scheme (they should all be zero) rprdsh: 0. rprddp: 0. evapcsh: 0. @@ -47,7 +47,7 @@ initial_conditions: sh_frac: 0. dp_frac: 0. dlf : 0. - #surface fluxes + #--surface fluxes bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml index d8d8e7694709..c26db8eb6814 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_mam4_aci_output +filename_prefix: p3_mam4_wetscav_output Averaging Type: Instant Field Names: - cldfrac_liq diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..b5a6be819ce2 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_cldfrac_p3_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 mam + LABELS shoc cld p3 physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 physics PEM mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml new file mode 100644 index 000000000000..f5f491a33807 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml @@ -0,0 +1,83 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic, mam4_wetscav] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables needed by mam4_wetscav + #--variables from convection scheme (they should all be zero) + rprdsh: 0. + rprddp: 0. + evapcsh: 0. + evapcdp: 0. + icwmrdp: 0. + icwmrsh: 0. + sh_frac: 0. + dp_frac: 0. + dlf : 0. + #--surface fluxes + bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + + #variable required for shoc + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variable required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml new file mode 100644 index 000000000000..8f7d04661aa4 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -0,0 +1,100 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cldfrac_p3_mam4_wetscav_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # wetscav + - T_mid + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 0ec3227dd7312ab040502093ce698e0a760e92c9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 4 Jul 2024 23:50:24 -0700 Subject: [PATCH 111/477] updated submodule to a newer version --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index f5700d5f2ac6..e949c1cb50b1 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit f5700d5f2ac6a6a7f845ee73a50a2bd1d268e22c +Subproject commit e949c1cb50b16eeafe0161caafc4c044171b7975 From d00cd5035485018a0d07cf69554b9be7945d85fc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 00:43:26 -0700 Subject: [PATCH 112/477] Cleans wetscav hpp file and minor changes to the cpp file --- .../eamxx_mam_wetscav_process_interface.cpp | 19 +--- .../eamxx_mam_wetscav_process_interface.hpp | 101 +++++++++--------- 2 files changed, 52 insertions(+), 68 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c581a5757064..1dc6265942c3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -1,19 +1,11 @@ #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" -#include - -#include "scream_config.h" // for SCREAM_CIME_BUILD - -// Remove the following<<<< -#include -#include -//>>>>>>> - /* ----------------------------------------------------------------- NOTES: 1. We should connect surface fluxes and add code to update the fluxes 2. Identify diagnostic variables and remove them from FM +3. Add assert statements to check output ranges */ namespace scream { @@ -27,12 +19,9 @@ MAMWetscav::MAMWetscav(const ekat::Comm &comm, */ } -AtmosphereProcessType MAMWetscav::type() const { - return AtmosphereProcessType::Physics; -} - -std::string MAMWetscav::name() const { return "mam4_wetscav"; } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMWetscav::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index 6f15752953cd..ea2de8681081 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -10,7 +10,6 @@ // For component name #include -#include #ifndef KOKKOS_ENABLE_CUDA #define protected_except_cuda public #define private_except_cuda public @@ -30,28 +29,21 @@ namespace scream { */ class MAMWetscav : public scream::AtmosphereProcess { - - using KT = ekat::KokkosTypes; - using view_1d = typename KT::template view_1d; + using KT = ekat::KokkosTypes; using view_2d = typename KT::template view_2d; - using view_3d = typename KT::template view_3d; - - using const_view_2d = typename KT::template view_2d; - using const_view_1d = typename KT::template view_1d; // remove it if possible - // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; public: - // Constructors + // Constructor MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The type of subcomponent - AtmosphereProcessType - type() const override; + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + // The name of the subcomponent - std::string name() const override; + std::string name() const { return "mam4_wetscav"; } // Set the grid and input output variables void set_grids( @@ -70,7 +62,6 @@ class MAMWetscav : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; - // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. @@ -79,10 +70,10 @@ class MAMWetscav : public scream::AtmosphereProcess { // on host: initializes preprocess functor with necessary state data void initialize(const int ncol_in, const int nlev_in, - const mam_coupling::WetAtmosphere& wet_atm_in, - const mam_coupling::AerosolState& wet_aero_in, - const mam_coupling::DryAtmosphere& dry_atm_in, - const mam_coupling::AerosolState& dry_aero_in) { + const mam_coupling::WetAtmosphere &wet_atm_in, + const mam_coupling::AerosolState &wet_aero_in, + const mam_coupling::DryAtmosphere &dry_atm_in, + const mam_coupling::AerosolState &dry_aero_in) { ncol_pre_ = ncol_in; nlev_pre_ = nlev_in; wet_atm_pre_ = wet_atm_in; @@ -92,26 +83,29 @@ class MAMWetscav : public scream::AtmosphereProcess { } KOKKOS_INLINE_FUNCTION - void operator()(const Kokkos::TeamPolicy::member_type& team) const { - const int i = team.league_rank(); // column index + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index // first, compute dry fields compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); team.team_barrier(); // second, we can use dry fields to compute dz, zmin, zint compute_vertical_layer_heights(team, dry_atm_pre_, i); compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - team.team_barrier(); // allows kernels below to use layer heights - } // operator() + // allows kernels below to use layer heights operator() + team.team_barrier(); + } - // number of horizontal columns and vertical levels + // Number of horizontal columns and vertical levels int ncol_pre_, nlev_pre_; - // local atmospheric and aerosol state data + // Local atmospheric and aerosol state data mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; - mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMWetscav::Preprocess + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMWetscav::Preprocess // Postprocessing functor struct Postprocess { @@ -119,24 +113,26 @@ class MAMWetscav : public scream::AtmosphereProcess { // on host: initializes postprocess functor with necessary state data void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere& wet_atm, - const mam_coupling::AerosolState& wet_aero, - const mam_coupling::DryAtmosphere& dry_atm, - const mam_coupling::AerosolState& dry_aero) { - ncol_post_ = ncol; - nlev_post_ = nlev; - wet_atm_post_ = wet_atm; + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; wet_aero_post_ = wet_aero; - dry_atm_post_ = dry_atm; + dry_atm_post_ = dry_atm; dry_aero_post_ = dry_aero; } KOKKOS_INLINE_FUNCTION - void operator()(const Kokkos::TeamPolicy::member_type& team) const { - const int i = team.league_rank(); // column index - compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, wet_aero_post_, i); + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); team.team_barrier(); - } // operator() + } // operator() // number of horizontal columns and vertical levels int ncol_post_, nlev_post_; @@ -144,33 +140,32 @@ class MAMWetscav : public scream::AtmosphereProcess { // local atmospheric and aerosol state data mam_coupling::WetAtmosphere wet_atm_post_; mam_coupling::DryAtmosphere dry_atm_post_; - mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; - }; // MAMWetscav::Postprocess - - - /* ----------------------------------------------- - * Local variables - * ------------------------------------------------ - */ + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // MAMWetscav::Postprocess + private: + // ----------------------------------------------- + // Local variables + // ------------------------------------------------ // pre- and postprocessing scratch pads (for wet <-> dry conversions) Preprocess preprocess_; Postprocess postprocess_; - // Number of horizontal columns and vertical levels int ncol_, nlev_; - //Number of aerosol modes + // Number of aerosol modes static constexpr int ntot_amode_ = mam4::AeroConfig::num_modes(); - // atmospheric variables + // Atmospheric variables mam_coupling::WetAtmosphere wet_atm_; mam_coupling::DryAtmosphere dry_atm_; - // wet dep + + // Work arrays view_2d work_; - // aerosol states - mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; + + // Aerosol states + mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; mam_coupling::Buffer buffer_; From dd7e21b66f666f06b679d974247a3578b1e72e82 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 01:32:23 -0700 Subject: [PATCH 113/477] Adds descriptive var names; removes conv vars from FM; fixes input and namelist yaml files with new var names --- .../cime_config/namelist_defaults_scream.xml | 37 ++---- .../eamxx_mam_wetscav_process_interface.cpp | 115 +++++++----------- .../eamxx_mam_wetscav_process_interface.hpp | 6 +- .../input.yaml | 47 ++----- .../mam/p3_mam4_wetscav/input.yaml | 26 ++-- .../mam/shoc_cldfrac_p3_wetscav/input.yaml | 26 ++-- .../single-process/mam/wet_scav/input.yaml | 26 ++-- 7 files changed, 94 insertions(+), 189 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 79fcc64f3848..0cd7f023bf14 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -561,35 +561,14 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0 - 0.5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 0.5 - 0.0 - 0.0 - 0.0 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 1dc6265942c3..768a1b827a44 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -53,7 +53,8 @@ void MAMWetscav::set_grids( m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); // layout for 2D (ncol, pcnst) - FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "pcnst"); + FieldLayout scalar2d_pconst = + m_grid->get_2d_vector_layout(pcnst, "num_phys_constants"); // -------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process @@ -101,10 +102,10 @@ void MAMWetscav::set_grids( //----------- Variables from microphysics scheme ------------- - // Evaporation from stratiform rain [kg/kg/s] (FIXME: Get it from P3) + // Evaporation from stratiform rain [kg/kg/s] add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); - // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) + // Stratiform rain production rate [kg/kg/s] add_field("precip_total_tend", scalar3d_mid, kg / kg / s, grid_name); @@ -116,38 +117,6 @@ void MAMWetscav::set_grids( // Total cloud fraction [fraction] add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); - //----------- Variables from convective scheme ------------- - - // Following variables are from convective parameterization (not implemented - // yet in EAMxx), so should be zero for now - - // Deep convective cloud fraction [fraction] - add_field("dp_frac", scalar3d_mid, nondim, grid_name); - - // Shallow convective cloud fraction [fraction] //NOT updated - add_field("sh_frac", scalar3d_mid, nondim, grid_name); - - // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] - add_field("evapcsh", scalar3d_mid, kg / kg / s, grid_name); - - // Evaporation rate of deep precipitation >=0. [kg/kg/s] - add_field("evapcdp", scalar3d_mid, kg / kg / s, grid_name); - - // Rain production, shallow convection [kg/kg/s] - add_field("rprdsh", scalar3d_mid, kg / kg / s, grid_name); - - // Rain production, deep convection [kg/kg/s] - add_field("rprddp", scalar3d_mid, kg / kg / s, grid_name); - - // In cloud water mixing ratio, deep convection [kg/kg] - add_field("icwmrdp", scalar3d_mid, kg / kg, grid_name); - - // In cloud water mixing ratio, shallow convection [kg/kg] - add_field("icwmrsh", scalar3d_mid, kg / kg, grid_name); - - // Detraining cld H20 from deep convection [kg/kg/s] - add_field("dlf", scalar3d_mid, kg / kg / s, grid_name); - // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- @@ -156,28 +125,32 @@ void MAMWetscav::set_grids( // for the land model // Wet deposition of hydrophilic black carbon [kg/m2/s] - add_field("bcphiwet", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_hydrophilic_bc", scalar3d_mid, kg / m2 / s, + grid_name); // Dry deposition of hydrophilic black carbon [kg/m2/s] - add_field("bcphidry", scalar3d_mid, kg / m2 / s, grid_name); + add_field("drydep_hydrophilic_bc", scalar3d_mid, kg / m2 / s, + grid_name); // Wet deposition of hydrophilic organic carbon [kg/m2/s] - add_field("ocphiwet", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_hydrophilic_oc", scalar3d_mid, kg / m2 / s, + grid_name); // Dry deposition of hydrophilic organic carbon [kg/m2/s] - add_field("ocphidry", scalar3d_mid, kg / m2 / s, grid_name); + add_field("drydep_hydrophilic_oc", scalar3d_mid, kg / m2 / s, + grid_name); // Wet deposition of dust (bin1) [kg/m2/s] - add_field("dstwet1", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_dust_bin1", scalar3d_mid, kg / m2 / s, grid_name); // Wet deposition of dust (bin2) [kg/m2/s] - add_field("dstwet2", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_dust_bin2", scalar3d_mid, kg / m2 / s, grid_name); // Wet deposition of dust (bin3) [kg/m2/s] - add_field("dstwet3", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_dust_bin3", scalar3d_mid, kg / m2 / s, grid_name); // Wet deposition of dust (bin4) [kg/m2/s] - add_field("dstwet4", scalar3d_mid, kg / m2 / s, grid_name); + add_field("wetdep_dust_bin4", scalar3d_mid, kg / m2 / s, grid_name); // Interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -193,11 +166,11 @@ void MAMWetscav::set_grids( "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios + // Note: Do *not* add cld borne aerosols to the "tracer" group as these are + // not advected const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(imode); - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { @@ -210,6 +183,8 @@ void MAMWetscav::set_grids( } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // Note: Do *not* add cld borne aerosols to the "tracer" group as these + // are not advected const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(imode, ispec); if(strlen(cld_mmr_field_name) > 0) { @@ -220,10 +195,6 @@ void MAMWetscav::set_grids( } } - // Tracers group -- do we need this in addition to the tracers above? In any - // case, this call should be idempotent, so it can't hurt. - add_group("tracers", grid_name, 1, Bundling::Required); - // The following fields are not needed by this process but we define them so // that we can create MAM4xx class objects like atmosphere, prognostics etc. @@ -260,14 +231,9 @@ void MAMWetscav::set_grids( add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); } -// ========================================================================================= -// ON HOST, returns the number of bytes of device memory needed by the above -// Buffer type given the number of columns and vertical levels -size_t MAMWetscav::requested_buffer_size_in_bytes() const { - return mam_coupling::buffer_size(ncol_, nlev_); -} - -// ========================================================================================= +// ================================================================ +// INIT_BUFFERS +// ================================================================ // ON HOST, initializeŃ• the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. @@ -282,7 +248,9 @@ void MAMWetscav::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMWetscav."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMWetscav::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- // Input fields read in from IC file, namelist or other processes @@ -401,7 +369,9 @@ void MAMWetscav::initialize_impl(const RunType run_type) { dry_aero_); } -// ========================================================================================= +// ================================================================ +// RUN_IMPL +// ================================================================ void MAMWetscav::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -425,31 +395,40 @@ void MAMWetscav::run_impl(const double dt) { // Following variables are from convective parameterization (not implemented // yet in EAMxx), so should be zero for now - auto sh_frac = get_field_in("sh_frac").get_view(); + auto sh_frac = view_2d("sh_frac", ncol_, nlev_); + Kokkos::deep_copy(sh_frac, 0); // Deep convective cloud fraction [fraction] - auto dp_frac = get_field_in("dp_frac").get_view(); + auto dp_frac = view_2d("dp_frac", ncol_, nlev_); + Kokkos::deep_copy(dp_frac, 0); // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] - auto evapcsh = get_field_in("evapcsh").get_view(); + auto evapcsh = view_2d("evapcsh", ncol_, nlev_); + Kokkos::deep_copy(evapcsh, 0); // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] - auto evapcdp = get_field_in("evapcdp").get_view(); + auto evapcdp = view_2d("evapcdp", ncol_, nlev_); + Kokkos::deep_copy(evapcdp, 0); // Rain production, shallow convection [kg/kg/s] - auto rprdsh = get_field_in("rprdsh").get_view(); + auto rprdsh = view_2d("rprdsh", ncol_, nlev_); + Kokkos::deep_copy(rprdsh, 0); // Rain production, deep convection [kg/kg/s] - auto rprddp = get_field_in("rprddp").get_view(); + auto rprddp = view_2d("rprddp", ncol_, nlev_); + Kokkos::deep_copy(rprddp, 0); // In cloud water mixing ratio, deep convection - auto icwmrdp = get_field_in("icwmrdp").get_view(); + auto icwmrdp = view_2d("icwmrdp", ncol_, nlev_); + Kokkos::deep_copy(icwmrdp, 0); // In cloud water mixing ratio, shallow convection - auto icwmrsh = get_field_in("icwmrsh").get_view(); + auto icwmrsh = view_2d("icwmrsh", ncol_, nlev_); + Kokkos::deep_copy(icwmrsh, 0); // Detraining cld H20 from deep convection [kg/kg/s] - auto dlf = get_field_in("dlf").get_view(); + auto dlf = view_2d("dlf", ncol_, nlev_); + Kokkos::deep_copy(dlf, 0); //----------- Variables from macrophysics scheme ------------- // Total cloud fraction diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index ea2de8681081..8bd81115d8a3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -50,7 +50,11 @@ class MAMWetscav : public scream::AtmosphereProcess { const std::shared_ptr grids_manager) override; // management of common atm process memory - size_t requested_buffer_size_in_bytes() const override; + // ON HOST, returns the number of bytes of device memory needed by the above + // Buffer type given the number of columns and vertical levels + size_t requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); + } void init_buffers(const ATMBufferManager &buffer_manager) override; // Initialize variables diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml index bfbec956d852..fc15d9ce2528 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -11,45 +11,18 @@ time_stepping: initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - surf_evap: 0.0 - surf_sens_flux: 0.0 - precip_liq_surf_mass: 0.0 - precip_ice_surf_mass: 0.0 - cldn: 0.5 - rprdsh: 1e-5 - rprddp: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 - cldt: 1e-5 - prain: 1e-5 - evapr: 1e-5 - icwmrdp: 1e-5 - icwmrsh: 1e-5 - rprddp: 1e-5 - rprdsh: 1e-5 - sh_frac: 1e-5 - dp_frac: 1e-5 - evapcsh: 1e-5 - evapcdp: 1e-5 - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] - dgncur_a: 1e-5 # aerosol particle diameter [m] - wetdens: 1e-5 # wet aerosol density [kg/m3] - qaerwat: 1e-5 # aerosol water [kg/kg] - dgnumwet: 1e-5 # wet aerosol diameter [m] - fracis: 1e-5 # fraction of transported species that are insoluble [fraction] - cldfrac_tot: 0.5 pbl_height: 25.0 phis : 0.1 - dlf : 0.0 - dp_ccf : 0.0 - sh_ccf : 0.0 + #variables needed by mam4_wetscav + #--surface fluxes + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] atmosphere_processes: atm_procs_list: [homme,physics] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml index f5d2dc036c0b..f0d9b7c89a3b 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml @@ -37,25 +37,15 @@ initial_conditions: pbl_height: 25.0 phis : 0.1 #variables needed by mam4_wetscav - #--variables from convection scheme (they should all be zero) - rprdsh: 0. - rprddp: 0. - evapcsh: 0. - evapcdp: 0. - icwmrdp: 0. - icwmrsh: 0. - sh_frac: 0. - dp_frac: 0. - dlf : 0. #--surface fluxes - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] #variable required for p3 precip_ice_surf_mass: 0.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml index f5f491a33807..804939a22058 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml @@ -49,25 +49,15 @@ initial_conditions: pbl_height: 25.0 phis : 0.1 #variables needed by mam4_wetscav - #--variables from convection scheme (they should all be zero) - rprdsh: 0. - rprddp: 0. - evapcsh: 0. - evapcdp: 0. - icwmrdp: 0. - icwmrsh: 0. - sh_frac: 0. - dp_frac: 0. - dlf : 0. #--surface fluxes - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] #variable required for shoc surf_sens_flux: 0.0 diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml index f2a62a15823e..d1fd9f3659da 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -30,25 +30,15 @@ initial_conditions: #variables from P3 precip_total_tend: 1e-5 nevapr: 1e-5 - #variables from convection scheme (they should all be zero) - rprdsh: 0. - rprddp: 0. - evapcsh: 0. - evapcdp: 0. - icwmrdp: 0. - icwmrsh: 0. - sh_frac: 0. - dp_frac: 0. - dlf : 0. #surface fluxes - bcphiwet: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] - bcphidry: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] - ocphiwet: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] - ocphidry: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] - dstwet1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] - dstwet2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] - dstwet3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] - dstwet4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From e6610dd5e5a4fdc657c7f39da0543f5e66b00778 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 01:42:56 -0700 Subject: [PATCH 114/477] Adds const for input vars, removes scorpio temp change --- .../eamxx_mam_wetscav_process_interface.cpp | 39 +++++++++---------- .../mam/impl/compute_particle_size.cpp | 9 ----- .../src/share/io/scream_scorpio_interface.cpp | 4 +- 3 files changed, 21 insertions(+), 31 deletions(-) delete mode 100644 components/eamxx/src/physics/mam/impl/compute_particle_size.cpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 768a1b827a44..8d4b6cda29c3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -439,11 +439,11 @@ void MAMWetscav::run_impl(const double dt) { // Evaporation from stratiform rain [kg/kg/s] auto nevapr = get_field_in("nevapr").get_view(); - // Stratiform rain production rate [kg/kg/s] (FIXME: Get it from P3) + // Stratiform rain production rate [kg/kg/s] auto prain = get_field_in("precip_total_tend").get_view(); - // ------------------------------------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------ // These variables are "Computed" or pure outputs for the process - // ------------------------------------------------------------------------------------------------------------------------- + // ------------------------------------------------------------------ const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); @@ -465,9 +465,9 @@ void MAMWetscav::run_impl(const double dt) { // Loop over atmosphere columns Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { - const int icol = team.league_rank(); // column index*/ + const int icol = team.league_rank(); // column index - auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + const auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); // set surface state data // fetch column-specific subviews into aerosol prognostics mam4::Prognostics progs = @@ -478,25 +478,24 @@ void MAMWetscav::run_impl(const double dt) { mam_coupling::interstitial_aerosols_tendencies_for_column( dry_aero_tends, icol); /// shallow_convective_precipitation_production - auto rprdsh_icol = ekat::subview(rprdsh, icol); + const auto rprdsh_icol = ekat::subview(rprdsh, icol); // deep_convective_precipitation_production - auto rprddp_icol = ekat::subview(rprddp, icol); + const auto rprddp_icol = ekat::subview(rprddp, icol); // deep_convective_precipitation_evaporation - auto evapcdp_icol = ekat::subview(evapcdp, icol); + const auto evapcdp_icol = ekat::subview(evapcdp, icol); // shallow_convective_precipitation_evaporation = - auto evapcsh_icol = ekat::subview(evapcsh, icol); + const auto evapcsh_icol = ekat::subview(evapcsh, icol); // deep_convective_cloud_fraction - auto dp_frac_icol = ekat::subview(dp_frac, icol); + const auto dp_frac_icol = ekat::subview(dp_frac, icol); // shallow_convective_cloud_fraction = - auto sh_frac_icol = ekat::subview(sh_frac, icol); - // FIXME: what is this? + const auto sh_frac_icol = ekat::subview(sh_frac, icol); - auto icwmrdp_col = ekat::subview(icwmrdp, icol); - auto icwmrsh_icol = ekat::subview(icwmrsh, icol); - auto nevapr_icol = ekat::subview(nevapr, icol); - auto cldt_icol = ekat::subview(cldt, icol); + const auto icwmrdp_col = ekat::subview(icwmrdp, icol); + const auto icwmrsh_icol = ekat::subview(icwmrsh, icol); + const auto nevapr_icol = ekat::subview(nevapr, icol); + const auto cldt_icol = ekat::subview(cldt, icol); - auto dlf_icol = ekat::subview(dlf, icol); + const auto dlf_icol = ekat::subview(dlf, icol); auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol); auto work_icol = ekat::subview(work, icol); @@ -504,9 +503,9 @@ void MAMWetscav::run_impl(const double dt) { ekat::subview(wet_geometric_mean_diameter_i, icol); auto dry_diameter_icol = ekat::subview(dry_geometric_mean_diameter_i, icol); - auto qaerwat_icol = ekat::subview(qaerwat, icol); - auto wetdens_icol = ekat::subview(wetdens, icol); - auto prain_icol = ekat::subview(prain, icol); + auto qaerwat_icol = ekat::subview(qaerwat, icol); + auto wetdens_icol = ekat::subview(wetdens, icol); + const auto prain_icol = ekat::subview(prain, icol); mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, diff --git a/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp b/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp deleted file mode 100644 index 06a4c4cb3bea..000000000000 --- a/components/eamxx/src/physics/mam/impl/compute_particle_size.cpp +++ /dev/null @@ -1,9 +0,0 @@ -namespace scream::impl { - -/*KOKKOS_INLINE_FUNCTION -void compute_particle_size(int icol, int nlev_, //in - view_1d& state_q, view_1d& qqcw) { - -}*/ - -} // namespace scream::impl diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 68a432819ec3..cb2b501f2b5c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - /*for (auto& it : s.files) { + for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - }*/ + } s.files.clear(); for (auto& it : s.decomps) { From 8c7d2a7ca862d7ff08d0cc9fbeea5f4923a4fb21 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 01:48:38 -0700 Subject: [PATCH 115/477] Cleans up namelist xml file and removes double ICs --- .../cime_config/namelist_defaults_scream.xml | 46 ++++--------------- 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 0cd7f023bf14..fb4dc5913d04 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -494,7 +494,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0,0.0 2.6e-08 - 0.0 + 0.0 0.0 0.0 0.0 @@ -519,6 +519,15 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0 + + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 0.0 @@ -534,41 +543,6 @@ be lost if SCREAM_HACK_XML is not enabled. 0 0.001 900.0 - - - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 0.0 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 - 1e-5 From 83a85eb310647d09c9de458ab5c0f2077e57ea7d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 02:00:46 -0700 Subject: [PATCH 116/477] Removes a left over comment --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 8d4b6cda29c3..c748432af134 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -188,8 +188,6 @@ void MAMWetscav::set_grids( const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(imode, ispec); if(strlen(cld_mmr_field_name) > 0) { - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these - // are NOT advected add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } From 140ee78b4495a2c08a22c7379454e08ec605802f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 12:55:33 -0700 Subject: [PATCH 117/477] Minor change in comments --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c748432af134..1a0d4b7412b9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -211,21 +211,22 @@ void MAMWetscav::set_grids( // Aerosol dry particle diameter [m] add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); - // wet aerosol density [kg/m3] + // Wet aerosol density [kg/m3] add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); - // aerosol water [kg/kg] + // Aerosol water [kg/kg] add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); - // wet aerosol diameter [m] + // Wet aerosol diameter [m] add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); - // fraction of transported species that are insoluble [fraction] + // Fraction of transported species that are insoluble [fraction] add_field("fracis", scalar3d_mid, nondim, grid_name); - // aerosol wet deposition (interstitial) [kg/m2/s] + // Aerosol wet deposition (interstitial) [kg/m2/s] add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); - // aerosol wet deposition (cloud water) [kg/m2/s] + + // Aerosol wet deposition (cloud water) [kg/m2/s] add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); } From aed9ffcc67688ad812873b0afaa72b5ded9965aa Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Fri, 5 Jul 2024 14:01:58 -0600 Subject: [PATCH 118/477] wet_scav - Removing Kokkos::resize. --- .../physics/mam/eamxx_mam_wetscav_process_interface.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 1a0d4b7412b9..bef4d6c7463d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -337,19 +337,18 @@ void MAMWetscav::initialize_impl(const RunType run_type) { //--------------------------------------------------------------------------------- // Allocate memory - // (Kokkos::resize only works on host to allocates memory) //--------------------------------------------------------------------------------- // Alllocate aerosol-related gas tendencies for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - Kokkos::resize(dry_aero_tends_.gas_mmr[g], ncol_, nlev_); + dry_aero_tends_.gas_mmr[g] = view_2d("gas_mmr",ncol_, nlev_); } // Allocate aerosol state tendencies (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - Kokkos::resize(dry_aero_tends_.int_aero_nmr[imode], ncol_, nlev_); + dry_aero_tends_.int_aero_nmr[imode] = view_2d("int_aero_nmr",ncol_, nlev_); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { - Kokkos::resize(dry_aero_tends_.int_aero_mmr[imode][ispec], ncol_, nlev_); + dry_aero_tends_.int_aero_mmr[imode][ispec] = view_2d("int_aero_mmr",ncol_, nlev_); } } From 019a54cc36a99d55e1657c6c7cb2ac8b0502b2db Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 16:31:13 -0700 Subject: [PATCH 119/477] Fixes some test fails: remove old output varnames, fixed output nc file name --- .../eamxx_mam_wetscav_process_interface.cpp | 7 +++--- .../input.yaml | 4 +++ .../output.yaml | 25 ------------------- .../mam/shoc_cldfrac_p3_wetscav/output.yaml | 2 +- externals/mam4xx | 2 +- 5 files changed, 10 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index bef4d6c7463d..20672e61f0f5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -340,15 +340,16 @@ void MAMWetscav::initialize_impl(const RunType run_type) { //--------------------------------------------------------------------------------- // Alllocate aerosol-related gas tendencies for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - dry_aero_tends_.gas_mmr[g] = view_2d("gas_mmr",ncol_, nlev_); + dry_aero_tends_.gas_mmr[g] = view_2d("gas_mmr", ncol_, nlev_); } // Allocate aerosol state tendencies (interstitial aerosols only) for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { - dry_aero_tends_.int_aero_nmr[imode] = view_2d("int_aero_nmr",ncol_, nlev_); + dry_aero_tends_.int_aero_nmr[imode] = view_2d("int_aero_nmr", ncol_, nlev_); for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { - dry_aero_tends_.int_aero_mmr[imode][ispec] = view_2d("int_aero_mmr",ncol_, nlev_); + dry_aero_tends_.int_aero_mmr[imode][ispec] = + view_2d("int_aero_mmr", ncol_, nlev_); } } diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml index fc15d9ce2528..c9b493a2958c 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -13,6 +13,10 @@ initial_conditions: topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} pbl_height: 25.0 phis : 0.1 + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_liq_surf_mass: 0.0 + precip_ice_surf_mass: 0.0 #variables needed by mam4_wetscav #--surface fluxes wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml index f85225d1274f..afe8437a2cf4 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml @@ -107,31 +107,6 @@ Fields: - dgncur_a - wetdens - qaerwat - - ptend_bc_a1 - - ptend_bc_a3 - - ptend_bc_a4 - - ptend_dst_a1 - - ptend_dst_a3 - - ptend_mom_a1 - - ptend_mom_a2 - - ptend_mom_a3 - - ptend_mom_a4 - - ptend_nacl_a1 - - ptend_nacl_a2 - - ptend_nacl_a3 - - ptend_num_a1 - - ptend_num_a2 - - ptend_num_a3 - - ptend_num_a4 - - ptend_pom_a1 - - ptend_pom_a3 - - ptend_pom_a4 - - ptend_so4_a1 - - ptend_so4_a2 - - ptend_so4_a3 - - ptend_soa_a1 - - ptend_soa_a2 - - ptend_soa_a3 - bc_c1 - bc_c3 - bc_c4 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml index 8f7d04661aa4..1ae53e6c1c30 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cldfrac_p3_mam4_wetscav_output +filename_prefix: shoc_cldfrac_p3_wetscav_output Averaging Type: Instant Field Names: # SHOC diff --git a/externals/mam4xx b/externals/mam4xx index e949c1cb50b1..cafdbc1b8813 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit e949c1cb50b16eeafe0161caafc4c044171b7975 +Subproject commit cafdbc1b88134038aff12c1cb00110c6397a4544 From fe66519678d9b1e718e493948d624e9983a5e71c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 16:36:26 -0700 Subject: [PATCH 120/477] Adds TODO to a comment as some editors can highlight these comments --- components/eamxx/src/share/io/scream_scorpio_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index cb2b501f2b5c..68a432819ec3 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - for (auto& it : s.files) { + /*for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - } + }*/ s.files.clear(); for (auto& it : s.decomps) { From af58453c13ea3ea0426d57f8a2844fcd3ba1dc44 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 16:49:32 -0700 Subject: [PATCH 121/477] Fixes previous commit: Adds a TODO comment here --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 2 +- components/eamxx/src/share/io/scream_scorpio_interface.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 20672e61f0f5..c6153794257a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -391,7 +391,7 @@ void MAMWetscav::run_impl(const double dt) { //----------- Variables from convective scheme ------------- - // Following variables are from convective parameterization (not implemented + // TODO: Following variables are from convective parameterization (not implemented // yet in EAMxx), so should be zero for now auto sh_frac = view_2d("sh_frac", ncol_, nlev_); diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 68a432819ec3..cb2b501f2b5c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - /*for (auto& it : s.files) { + for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - }*/ + } s.files.clear(); for (auto& it : s.decomps) { From 4d3b7bc26ccf11cfda080c80a1464cc65cb3d184 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 18:01:22 -0700 Subject: [PATCH 122/477] Adds a new function to add additional tracers for MAM4xx --- .../testmods_dirs/scream/mam4xx/aci/shell_commands | 8 +++++--- .../scream/mam4xx/optics/shell_commands | 9 +++++---- .../scream/mam4xx/wetscav/shell_commands | 12 ++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index 68eaf51698b3..658c94a3cf67 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -1,7 +1,9 @@ -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh #modify initial condition file to get aerosol species ICs $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands index 1c22bd9ee454..224c16586104 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands @@ -1,7 +1,8 @@ - -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_optics,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands index f0e96c413985..97f442f88d85 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -1,7 +1,11 @@ +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" - +#------------------------------------------------------ +#Update IC file and add wetscav process +#------------------------------------------------------ $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_wetscav,rrtmgp" -b From 6de6c33682b86c3d4caae1a2ac6a18879e4b53fc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 5 Jul 2024 18:02:08 -0700 Subject: [PATCH 123/477] Adds the new script for computing number of tracers --- .../scream/mam4xx/update_eamxx_num_tracers.sh | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh new file mode 100644 index 000000000000..930af526eb8d --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh @@ -0,0 +1,22 @@ +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +# Additional MAM4xx tracers (MAM4xx adds 31 tracers) +ADDITIONAL_MAM4xx_TRACERS=31 + +# Original CMAKE options in env_build.xml +orig_cmake_opt=`./xmlquery --value SCREAM_CMAKE_OPTIONS` + +# Extract the number of tracers +orig_tracer_num=$(echo $orig_cmake_opt | grep -oP 'SCREAM_NUM_TRACERS \K[0-9]+') + +# Update number of tracers +new_tracer_num=$((orig_tracer_num + ADDITIONAL_MAM4xx_TRACERS)) + +# Form the new CMake options string by replacing the original number with the new number +new_cmake_opt=$(echo $orig_cmake_opt | sed "s/SCREAM_NUM_TRACERS $orig_tracer_num/SCREAM_NUM_TRACERS $new_tracer_num/") + +# Update cmake options string +./xmlchange SCREAM_CMAKE_OPTIONS="$new_cmake_opt" \ No newline at end of file From 54f71108424698eb2d78d89a88c4119ce30c69a2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 6 Jul 2024 20:56:14 -0700 Subject: [PATCH 124/477] Fixes script to update tracer number in scream cmake options --- .../testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh | 6 ++++-- .../testmods_dirs/scream/mam4xx/wetscav/shell_commands | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) mode change 100644 => 100755 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh old mode 100644 new mode 100755 index 930af526eb8d..c533123847ef --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh @@ -1,5 +1,7 @@ +#!/bin/sh + #------------------------------------------------------ -# MAM4xx adds additionaltracers to the simulation +# MAM4xx adds additional tracers to the simulation # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ @@ -19,4 +21,4 @@ new_tracer_num=$((orig_tracer_num + ADDITIONAL_MAM4xx_TRACERS)) new_cmake_opt=$(echo $orig_cmake_opt | sed "s/SCREAM_NUM_TRACERS $orig_tracer_num/SCREAM_NUM_TRACERS $new_tracer_num/") # Update cmake options string -./xmlchange SCREAM_CMAKE_OPTIONS="$new_cmake_opt" \ No newline at end of file +`./xmlchange SCREAM_CMAKE_OPTIONS="$new_cmake_opt"` diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands index 97f442f88d85..7b5b9a876503 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -1,8 +1,10 @@ +#!/bin/sh #------------------------------------------------------ # MAM4xx adds additionaltracers to the simulation # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ #Update IC file and add wetscav process From ffffd51a7f0808d8c1471198f984d246fb74b7b6 Mon Sep 17 00:00:00 2001 From: noel Date: Wed, 3 Jul 2024 15:38:51 -0700 Subject: [PATCH 125/477] update module versions on pm-gpu --- cime_config/machines/config_machines.xml | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 5c6caed992c2..4a988f9f5402 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -368,6 +368,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -378,24 +379,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/22.7 + nvidia/24.5 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - gcc-mixed/11.2.0 + gcc-native-mixed/12.3 @@ -407,12 +408,12 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + cray-libsci/23.12.5 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 From 135190ec84df237b145f2e3375fac7872f04b671 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 8 Jul 2024 16:48:39 -0700 Subject: [PATCH 126/477] Updating muller-gpu module versions to be similar as that of pm-gpu --- cime_config/machines/config_machines.xml | 26 ++++++++---------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 4a988f9f5402..c04dff5976cd 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -678,6 +678,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -688,26 +689,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 - + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 - cudatoolkit/11.7 - + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 + gcc-native-mixed/12.3 @@ -719,20 +718,13 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 - evp-patch From dd2b0c9559b1c03116dcb78ae9c35506a66b8e93 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 9 Jul 2024 07:17:24 -0600 Subject: [PATCH 127/477] Fix some input/output arrays as suggested by Balwinder. --- ...x_mam_dry_deposition_process_interface.cpp | 108 +++++++++--------- ...x_mam_dry_deposition_process_interface.hpp | 21 ++-- .../single-process/mam/drydep/input.yaml | 14 +-- 3 files changed, 74 insertions(+), 69 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 2968b9927ef5..bb61390c6b7a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -16,27 +16,6 @@ MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. */ - std::string nname = "Obukhov_length"; - if(params.isParameter(nname)) - parameters_.Obukhov_length_ = params.get(nname); - nname = "surface_friction_velocty"; - if(params.isParameter(nname)) - parameters_.surface_friction_velocty_ = params.get(nname); - nname = "land_fraction"; - if(params.isParameter(nname)) - parameters_.land_fraction_ = params.get(nname); - nname = "ice_fraction"; - if(params.isParameter(nname)) - parameters_.ice_fraction_ = params.get(nname); - nname = "ocean_fraction"; - if(params.isParameter(nname)) - parameters_.ocean_fraction_ = params.get(nname); - nname = "friction_velocity"; - if(params.isParameter(nname)) - parameters_.friction_velocity_ = params.get(nname); - nname = "aerodynamical_resistance"; - if(params.isParameter(nname)) - parameters_.aerodynamical_resistance_ = params.get(nname); } // ========================================================================================= @@ -79,14 +58,16 @@ void MAMDryDep::set_grids( // Layout for tracers. const int pcnst = mam4::aero_model::pcnst; FieldLayout scalar4d_layout_q{ - {COL, LEV, NMODES}, {ncol_, nlev_, pcnst}}; + {COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; FieldLayout scalar4d_layout_qqcw_tends{ - {COL, NMODES, LEV}, {ncol_, pcnst, nlev_}}; + {COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; - // Memory to format teh wet_aero.int_aero_nmr and wet_aero.int_aero_mmr + // Tempary memory to format the wet_aero.int_aero_nmr and wet_aero.int_aero_mmr // views into the order expected by mam4xx. add_field("qtracers", scalar4d_layout_q, kg / kg, grid_name); add_field("d_qtracers_dt", scalar4d_layout_q, kg / kg / s, grid_name); + + // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] @@ -114,23 +95,21 @@ void MAMDryDep::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + // add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); // TODO: The following are not used by drydep but to create a dry atmosphere object. add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - // - // TODO: Currently the following are scalar parameters that are fixed - // for the whole grid. Do these need to be column specific? - // add_field("obklen", scalar2d_layout, m, grid_name); - // add_field("surfric", scalar2d_layout, m / s, grid_name); - // add_field("landfrac", scalar2d_layout, nondim, grid_name); - // add_field("icefrac", scalar2d_layout, nondim, grid_name); - // add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - // add_field("fv", scalar2d_layout, m / s, grid_name); - // add_field("ram1", scalar2d_layout, s / m, grid_name); + + add_field("Obukhov_length", scalar2d_layout, m, grid_name); + add_field("surface_friction_velocty", scalar2d_layout, m / s, grid_name); + add_field("land_fraction", scalar2d_layout, nondim, grid_name); + add_field("ice_fraction", scalar2d_layout, nondim, grid_name); + add_field("ocean_fraction", scalar2d_layout, nondim, grid_name); + add_field("friction_velocity", scalar2d_layout, m / s, grid_name); + add_field("aerodynamical_resistance", scalar2d_layout, s / m, grid_name); // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -212,6 +191,15 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.p_del = get_field_in("pseudo_density").get_view(); dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); + + obklen_ = get_field_in("Obukhov_length").get_view(); + surfric_ = get_field_in("surface_friction_velocty").get_view(); + landfrac_ = get_field_in("land_fraction").get_view(); + icefrac_ = get_field_in("ice_fraction").get_view(); + ocnfrac_ = get_field_in("ocean_fraction").get_view(); + friction_velocity_ = get_field_in("friction_velocity").get_view(); + aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.z_iface = buffer_.z_iface; @@ -227,8 +215,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dgncur_awet_ = get_field_out("dgncur_awet").get_view(); wet_dens_ = get_field_out("wetdens").get_view(); - qtracers_ = get_field_out("qtracers").get_view(); - d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + qtracers_ = get_field_out("qtracers").get_view(); + d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols").get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols").get_view(); tendencies_ = get_field_out("Tendencies").get_view(); @@ -282,11 +271,17 @@ namespace { KOKKOS_INLINE_FUNCTION void compute_tendencies( const MAMDryDep::KT::MemberType &team, - const MAMDryDep::Parameters ¶meters, const mam4::DryDeposition &dry_deposition, const double dt, - MAMDryDep::view_3d qtracers, - MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::const_view_1d obklen, + const MAMDryDep::const_view_1d surfric, + const MAMDryDep::const_view_1d landfrac, + const MAMDryDep::const_view_1d icefrac, + const MAMDryDep::const_view_1d ocnfrac, + const MAMDryDep::const_view_1d friction_velocity, + const MAMDryDep::const_view_1d aerodynamical_resistance, + const MAMDryDep::view_3d qtracers, + const MAMDryDep::view_3d d_qtracers_dt, const MAMDryDep::view_2d dgncur_awet[mam_coupling::num_aero_modes()], const MAMDryDep::view_2d wet_dens[mam_coupling::num_aero_modes()], const mam_coupling::DryAtmosphere &dry_atm, @@ -317,13 +312,14 @@ void compute_tendencies( for (int i=0; i; using view_3d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; using const_view_2d = Field::view_dev_t; using const_view_3d = Field::view_dev_t; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); @@ -42,11 +43,19 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_3d d_qtracers_dt_; view_3d dgncur_awet_; view_3d wet_dens_; - view_3d tendencies_; + view_3d tendencies_; view_2d aerdepdrycw_; view_2d aerdepdryis_; + const_view_1d obklen_; + const_view_1d surfric_; + const_view_1d landfrac_; + const_view_1d icefrac_; + const_view_1d ocnfrac_; + const_view_1d friction_velocity_; + const_view_1d aerodynamical_resistance_; + public: using KT = ekat::KokkosTypes; @@ -121,19 +130,9 @@ class MAMDryDep final : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess - struct Parameters { - Real Obukhov_length_ = 0.20723257141035126e+03; - Real surface_friction_velocty_ = 0.39900396673305327; - Real land_fraction_ = 0.1; - Real ice_fraction_ = 0.6; - Real ocean_fraction_ = 0.3; - Real friction_velocity_ = 0.46637129718055864; - Real aerodynamical_resistance_ = 0.91147859222259044e+02; - }; private: // pre- and postprocessing scratch pads Preprocess preprocess_; - Parameters parameters_; }; // MAMDryDep } // namespace scream diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 1163046ccce5..8b110dc1634d 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -27,13 +27,13 @@ initial_conditions: T_mid: 273.0 dgncur_awet: 1e-5 wetdens: 1e-5 - obklen: 1e-5 - surfric: 1e-5 - landfrac: 1.0 - icefrac: 0.0 - ocnfrac: 0.0 - fv: 1e-5 - ram1: 1e-5 + Obukhov_length: 1e-5 + surface_friction_velocty: 1e-5 + land_fraction: 1.0 + ice_fraction: 0.0 + ocean_fraction: 0.0 + friction_velocity: 1e-5 + aerodynamical_resistance: 1e-5 w_updraft: 1e-5 pbl_height: 0.0 From 7197f506d95ba203c3e9e27412b1e67870cd0c21 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 10 Jul 2024 09:44:44 -0600 Subject: [PATCH 128/477] More code cleanup. --- ...x_mam_dry_deposition_process_interface.cpp | 230 +++++++++--------- ...x_mam_dry_deposition_process_interface.hpp | 11 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- 3 files changed, 115 insertions(+), 128 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index bb61390c6b7a..eced2cf49643 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -3,9 +3,50 @@ #include "mam4xx/drydep.hpp" /* -Future work: -Wirte comments -write in/outs for all variables clearly +Inputs: + +Atmosphere: + temperature "T_mid" + pressure "p_mid" + interface_pressure "p_int" + hydrostatic_dp "pseudo_density" + +Diagnostics: + tracer_mixing_ratio "qtracers" + wet_geometric_mean_diameter_i "dgncur_awet" + wet_density "wetdens" + +Diagnostic Scalar Parameters, one per column: + "Obukhov_length" + "surface_friction_velocty" + "land_fraction" + "ice_fraction" + "ocean_fraction" + "friction_velocity" + "aerodynamical_resistance" + +Prognostics: + n_mode_c "mam_coupling::cld_aero_nmr_field_name(m)" + q_aero_c "mam_coupling::int_aero_nmr_field_name(m)" + + + +Outputs: + +Diagnostics: + d_tracer_mixing_ratio_dt "d_qtracers_dt" + deposition_flux_of_cloud_borne_aerosols "deposition_flux_of_cloud_borne_aerosols" + deposition_flux_of_interstitial_aerosols "deposition_flux_of_interstitial_aerosols" + +Computed internally, could be exposed if needed by another process + vlc_grv(nlev) : dep velocity of gravitational settling [m/s] + vlc_trb(nlev) : dep velocity of turbulent dry deposition [m/s] + vlc_dry(nlev) : dep velocity, sum of vlc_grv and vlc_trb [m/s] + +Tendencies: + n_mode_c "Tendencies" + q_aero_c "Tendencies" + */ namespace scream { @@ -31,6 +72,8 @@ void MAMDryDep::set_grids( auto m3 = m * m * m; // meter cubed + auto nondim = ekat::units::Units::nondimensional(); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); @@ -40,8 +83,6 @@ void MAMDryDep::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - auto nondim = ekat::units::Units::nondimensional(); - // Layout for 2D (2d horiz) variable const FieldLayout scalar2d_layout{{COL}, {ncol_}}; @@ -51,9 +92,10 @@ void MAMDryDep::set_grids( const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + // at mid points const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ - {NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; // mid points + {NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; // Layout for tracers. const int pcnst = mam4::aero_model::pcnst; @@ -95,7 +137,7 @@ void MAMDryDep::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - // + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); @@ -192,13 +234,13 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); - obklen_ = get_field_in("Obukhov_length").get_view(); - surfric_ = get_field_in("surface_friction_velocty").get_view(); - landfrac_ = get_field_in("land_fraction").get_view(); - icefrac_ = get_field_in("ice_fraction").get_view(); - ocnfrac_ = get_field_in("ocean_fraction").get_view(); - friction_velocity_ = get_field_in("friction_velocity").get_view(); - aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); + obukhov_length_ = get_field_in("Obukhov_length").get_view(); + surface_friction_velocty_ = get_field_in("surface_friction_velocty").get_view(); + land_fraction_ = get_field_in("land_fraction").get_view(); + ice_fraction_ = get_field_in("ice_fraction").get_view(); + ocean_fraction_ = get_field_in("ocean_fraction").get_view(); + friction_velocity_ = get_field_in("friction_velocity").get_view(); + aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.z_mid = buffer_.z_mid; @@ -268,74 +310,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_, dry_aero_); } namespace { -KOKKOS_INLINE_FUNCTION -void compute_tendencies( - const MAMDryDep::KT::MemberType &team, - const mam4::DryDeposition &dry_deposition, - const double dt, - const MAMDryDep::const_view_1d obklen, - const MAMDryDep::const_view_1d surfric, - const MAMDryDep::const_view_1d landfrac, - const MAMDryDep::const_view_1d icefrac, - const MAMDryDep::const_view_1d ocnfrac, - const MAMDryDep::const_view_1d friction_velocity, - const MAMDryDep::const_view_1d aerodynamical_resistance, - const MAMDryDep::view_3d qtracers, - const MAMDryDep::view_3d d_qtracers_dt, - const MAMDryDep::view_2d dgncur_awet[mam_coupling::num_aero_modes()], - const MAMDryDep::view_2d wet_dens[mam_coupling::num_aero_modes()], - const mam_coupling::DryAtmosphere &dry_atm, - const mam_coupling::AerosolState &dry_aero, - const mam_coupling::AerosolState &wet_aero, - MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, - MAMDryDep::view_3d tendencies) -{ - const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); - const Real t = 0; - const int icol = team.league_rank(); - - const mam4::AeroConfig aero_config; - mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); - mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::Surface surf; - mam4::Diagnostics diags; - mam4::Tendencies tends; - - diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); - diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - - for (int i=0; i::get_default_team_policy(ncol, nlev); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { const int num_aero_modes = mam_coupling::num_aero_modes(); - MAMDryDep::view_2d dgncur_awet[num_aero_modes]; - MAMDryDep::view_2d wet_dens[num_aero_modes]; - - for (int i=0; i::get_default_team_policy(ncol, nlev); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { - const int column = team.league_rank(); - compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, column); + const int num_aero_species = mam_coupling::num_aero_species(); + const int icol = team.league_rank(); + const Real t = 0; + compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); + team.team_barrier(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int lev) { for (int mode = 0; mode < num_aero_modes; ++mode) { int icnst = mam4::ConvProc::numptrcw_amode(mode); - qtracers(column, lev, icnst) = wet_aero.int_aero_nmr[mode](column,lev); + qtracers(icol, lev, icnst) = wet_aero.int_aero_nmr[mode](icol,lev); for (int species = 0; species < num_aero_species; ++species) { icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); if (-1 < icnst) { - qtracers(column, lev, icnst) = wet_aero.int_aero_mmr[mode][species](column,lev); + qtracers(icol, lev, icnst) = wet_aero.int_aero_mmr[mode][species](icol,lev); } } } }); + team.team_barrier(); + + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + mam4::Surface surf; + mam4::Diagnostics diags; + mam4::Tendencies tends; + + for (int i=0; i(gas_mmr_names(gas)); + return gas_mmr_names(gas); } // This type stores multi-column views related specifically to the wet From 476a38c098fbf34c9828fe55476bf60a4dd3a40d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 06:27:56 -0700 Subject: [PATCH 129/477] Moves around code to match previous mam4xx processes --- ...x_mam_dry_deposition_process_interface.cpp | 460 ++++++++++-------- ...x_mam_dry_deposition_process_interface.hpp | 29 +- .../single-process/mam/drydep/input.yaml | 1 + 3 files changed, 273 insertions(+), 217 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index eced2cf49643..aa0f643b974a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -35,8 +35,10 @@ Diagnostic Scalar Parameters, one per column: Diagnostics: d_tracer_mixing_ratio_dt "d_qtracers_dt" - deposition_flux_of_cloud_borne_aerosols "deposition_flux_of_cloud_borne_aerosols" - deposition_flux_of_interstitial_aerosols "deposition_flux_of_interstitial_aerosols" + deposition_flux_of_cloud_borne_aerosols +"deposition_flux_of_cloud_borne_aerosols" + deposition_flux_of_interstitial_aerosols +"deposition_flux_of_interstitial_aerosols" Computed internally, could be exposed if needed by another process vlc_grv(nlev) : dep velocity of gravitational settling [m/s] @@ -51,7 +53,6 @@ Computed internally, could be exposed if needed by another process namespace scream { -// ========================================================================================= MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be @@ -59,22 +60,18 @@ MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) */ } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMDryDep::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - - auto m3 = m * m * m; // meter cubed - - auto nondim = ekat::units::Units::nondimensional(); + // set grid for all the inputs and outputs + // use physics grid + grid_ = grids_manager->get_grid("Physics"); - grid_ = grids_manager->get_grid("Physics"); + // Name of the grid const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -83,90 +80,139 @@ void MAMDryDep::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; - + // Layout for 2D (2d horiz) variable + // FIXME: Fix layouts based on new format + const FieldLayout scalar2d{{COL}, {ncol_}}; + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + const FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; + const FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ - {NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; + FieldLayout scalar4d_mid{{NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; // Layout for tracers. const int pcnst = mam4::aero_model::pcnst; - FieldLayout scalar4d_layout_q{ - {COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; - FieldLayout scalar4d_layout_qqcw_tends{ - {COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; + FieldLayout scalar4d_q{{COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; + FieldLayout scalar4d_qqcw_tends{{COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; + + using namespace ekat::units; + + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + auto nondim = ekat::units::Units::nondimensional(); + + auto m3 = m * m * m; // meter cubed + + // -------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg] + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_mid, Pa / s, grid_name); - // Tempary memory to format the wet_aero.int_aero_nmr and wet_aero.int_aero_mmr - // views into the order expected by mam4xx. - add_field("qtracers", scalar4d_layout_q, kg / kg, grid_name); - add_field("d_qtracers_dt", scalar4d_layout_q, kg / kg / s, grid_name); + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + // Planetary boundary layer height [m] + add_field("pbl_height", scalar2d, m, grid_name); + + static constexpr auto m2 = m * m; + static constexpr auto s2 = s * s; + + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Total cloud fraction [fraction] + // FIXME: Is is cldfrac_liq instead? find out + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + + // Obukhov length [m] + add_field("Obukhov_length", scalar2d, m, grid_name); + + // Surface friction velocty [m] + add_field("surface_friction_velocty", scalar2d, m / s, grid_name); + + // Land fraction [fraction] + add_field("land_fraction", scalar2d, nondim, grid_name); + + // Ice fraction [unitless] + add_field("ice_fraction", scalar2d, nondim, grid_name); + + // Ocean fraction [unitless] + add_field("ocean_fraction", scalar2d, nondim, grid_name); + + // Friction velocity from land model [m/s] + add_field("friction_velocity", scalar2d, m / s, grid_name); + + // Aerodynamical resistance from land model [s/m] + add_field("aerodynamical_resistance", scalar2d, s / m, grid_name); + + // Tempary memory to format the wet_aero.int_aero_nmr and + // wet_aero.int_aero_mmr views into the order expected by mam4xx. + add_field("qtracers", scalar4d_q, kg / kg, grid_name); + add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); + add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, + 1 / (m * m) / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_interstitial_aerosols", scalar3d_layout_mid, 1/(m*m)/s, grid_name); - add_field("Tendencies", scalar4d_layout_qqcw_tends, kg / kg / s, grid_name); + add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, + 1 / (m * m) / s, grid_name); + add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, + grid_name); - // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio - - add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); - - // TODO: The following are not used by drydep but to create a dry atmosphere object. - add_field("pbl_height", scalar2d_layout, m, grid_name); // planetary boundary layer height - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction - - add_field("Obukhov_length", scalar2d_layout, m, grid_name); - add_field("surface_friction_velocty", scalar2d_layout, m / s, grid_name); - add_field("land_fraction", scalar2d_layout, nondim, grid_name); - add_field("ice_fraction", scalar2d_layout, nondim, grid_name); - add_field("ocean_fraction", scalar2d_layout, nondim, grid_name); - add_field("friction_velocity", scalar2d_layout, m / s, grid_name); - add_field("aerodynamical_resistance", scalar2d_layout, s / m, grid_name); + add_field("wetdens", scalar4d_mid, kg / m3, grid_name); + add_field("dgncur_awet", scalar4d_mid, m, grid_name); + + // TODO: The following are not used by drydep but to create a dry atmosphere + // object. // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for(int m = 0; m < num_aero_modes; ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); } } } @@ -174,15 +220,13 @@ void MAMDryDep::set_grids( for(int m = 0; m < num_aero_modes; ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } } @@ -190,9 +234,9 @@ void MAMDryDep::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } } // ========================================================================================= @@ -222,47 +266,53 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? - dry_atm_.pblh = get_field_in("pbl_height").get_view(); - - obukhov_length_ = get_field_in("Obukhov_length").get_view(); - surface_friction_velocty_ = get_field_in("surface_friction_velocty").get_view(); - land_fraction_ = get_field_in("land_fraction").get_view(); - ice_fraction_ = get_field_in("ice_fraction").get_view(); - ocean_fraction_ = get_field_in("ocean_fraction").get_view(); - friction_velocity_ = get_field_in("friction_velocity").get_view(); - aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); - - dry_atm_.p_int = get_field_in("p_int").get_view(); - dry_atm_.z_mid = buffer_.z_mid; - dry_atm_.z_iface = buffer_.z_iface; - dry_atm_.dz = buffer_.dz; - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot") + .get_view(); // FIXME: tot or liq? + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + + obukhov_length_ = get_field_in("Obukhov_length").get_view(); + surface_friction_velocty_ = + get_field_in("surface_friction_velocty").get_view(); + land_fraction_ = get_field_in("land_fraction").get_view(); + ice_fraction_ = get_field_in("ice_fraction").get_view(); + ocean_fraction_ = get_field_in("ocean_fraction").get_view(); + friction_velocity_ = + get_field_in("friction_velocity").get_view(); + aerodynamical_resistance_ = + get_field_in("aerodynamical_resistance").get_view(); + + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; dry_atm_.w_updraft = buffer_.w_updraft; - dry_atm_.z_surf = 0.0; // FIXME: for now + dry_atm_.z_surf = 0.0; // FIXME: for now dgncur_awet_ = get_field_out("dgncur_awet").get_view(); - wet_dens_ = get_field_out("wetdens").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); - qtracers_ = get_field_out("qtracers").get_view(); - d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + qtracers_ = get_field_out("qtracers").get_view(); + d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); - aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols").get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols").get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + .get_view(); + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + .get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -306,99 +356,101 @@ void MAMDryDep::initialize_impl(const RunType run_type) { } // set up our preprocess functor - preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, - dry_atm_, dry_aero_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } namespace { void compute_tendencies( - const int ncol, - const int nlev, - const mam4::DryDeposition dry_deposition, - const double dt, - const MAMDryDep::const_view_1d obklen, + const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, + const double dt, const MAMDryDep::const_view_1d obklen, const MAMDryDep::const_view_1d surfric, const MAMDryDep::const_view_1d landfrac, const MAMDryDep::const_view_1d icefrac, const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, - MAMDryDep::view_3d d_qtracers_dt, - const MAMDryDep::view_3d dgncur_awet_, - const MAMDryDep::view_3d wet_dens_, + MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, - const mam_coupling::AerosolState wet_aero, - MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, - MAMDryDep::view_3d tendencies) -{ - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlev); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType& team) { - const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); - const int icol = team.league_rank(); - const Real t = 0; - - compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); - team.team_barrier(); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int lev) { - for (int mode = 0; mode < num_aero_modes; ++mode) { - int icnst = mam4::ConvProc::numptrcw_amode(mode); - qtracers(icol, lev, icnst) = wet_aero.int_aero_nmr[mode](icol,lev); - for (int species = 0; species < num_aero_species; ++species) { - icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); - if (-1 < icnst) { - qtracers(icol, lev, icnst) = wet_aero.int_aero_mmr[mode][species](icol,lev); - } - } - } - }); - team.team_barrier(); - - mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); - mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::Surface surf; - mam4::Diagnostics diags; - mam4::Tendencies tends; - - for (int i=0; i::get_default_team_policy( + ncol, nlev); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); + const int icol = team.league_rank(); + const Real t = 0; + + compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); + team.team_barrier(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { + for(int mode = 0; mode < num_aero_modes; ++mode) { + int icnst = mam4::ConvProc::numptrcw_amode(mode); + qtracers(icol, lev, icnst) = + wet_aero.int_aero_nmr[mode](icol, lev); + for(int species = 0; species < num_aero_species; ++species) { + icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); + if(-1 < icnst) { + qtracers(icol, lev, icnst) = + wet_aero.int_aero_mmr[mode][species](icol, lev); + } + } + } + }); + team.team_barrier(); + + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + mam4::Surface surf; + mam4::Diagnostics diags; + mam4::Tendencies tends; + + for(int i = 0; i < num_aero_modes; ++i) { + diags.wet_geometric_mean_diameter_i[i] = + ekat::subview(dgncur_awet_, i, icol); + diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); + } + diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); + diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); + diags.deposition_flux_of_cloud_borne_aerosols = + ekat::subview(aerdepdrycw, icol); + diags.deposition_flux_of_interstitial_aerosols = + ekat::subview(aerdepdryis, icol); + + diags.Obukhov_length = obklen[icol]; + diags.surface_friction_velocty = surfric[icol]; + diags.land_fraction = landfrac[icol]; + diags.ice_fraction = icefrac[icol]; + diags.ocean_fraction = ocnfrac[icol]; + diags.friction_velocity = friction_velocity[icol]; + diags.aerodynamical_resistance = aerodynamical_resistance[icol]; + + // Fill Tendency views + for(int m = 0; m < num_aero_modes; ++m) { + int iconv = mam4::ConvProc::numptrcw_amode(m); + tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); + for(int a = 0; a < num_aero_species; ++a) { + iconv = mam4::ConvProc::lmassptrcw_amode(a, m); + if(-1 < iconv) + tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); + } + } + + const mam4::AeroConfig aero_config; + dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, + progs, diags, tends); + }); } +} // namespace // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - using DryDep = mam4::DryDeposition; + using DryDep = mam4::DryDeposition; const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -414,27 +466,27 @@ void MAMDryDep::run_impl(const double dt) { // for (int i=0; i::get_default_team_policy(ncol_, nlev_); + // const auto policy = + // ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); // Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { // dry_deposition.validate( aero_config, team, atm, progs); // } // Kokkos::fence(); - compute_tendencies(ncol_, nlev_, dry_deposition, dt, - obukhov_length_, surface_friction_velocty_, - land_fraction_, ice_fraction_, ocean_fraction_, - friction_velocity_, aerodynamical_resistance_, - qtracers_, d_qtracers_dt_, // d_qtracers_dt_ is an output - dgncur_awet_, wet_dens_, - dry_atm_, dry_aero_, wet_aero_, - // Outputs: - aerdepdrycw_, aerdepdryis_, tendencies_); + compute_tendencies(ncol_, nlev_, dry_deposition, dt, obukhov_length_, + surface_friction_velocty_, land_fraction_, ice_fraction_, + ocean_fraction_, friction_velocity_, + aerodynamical_resistance_, qtracers_, + d_qtracers_dt_, // d_qtracers_dt_ is an output + dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, + // Outputs: + aerdepdrycw_, aerdepdryis_, tendencies_); Kokkos::fence(); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 418bea8fe973..1144bd79ff68 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -15,18 +15,19 @@ namespace scream { // The process responsible for handling MAM4 dry deposition. The AD // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { -public: - using view_2d = Field::view_dev_t; - using view_3d = Field::view_dev_t; - using const_view_1d = Field::view_dev_t; - using const_view_2d = Field::view_dev_t; - using const_view_3d = Field::view_dev_t; + public: + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); -private: + + private: // number of horizontal columns and vertical levels int ncol_, nlev_; - //Wet and dry states of atmosphere + // Wet and dry states of atmosphere mam_coupling::WetAtmosphere wet_atm_; mam_coupling::DryAtmosphere dry_atm_; @@ -56,7 +57,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { const_view_1d friction_velocity_; const_view_1d aerodynamical_resistance_; -public: + public: using KT = ekat::KokkosTypes; // Constructor @@ -67,10 +68,12 @@ class MAMDryDep final : public scream::AtmosphereProcess { // -------------------------------------------------------------------------- // The type of subcomponent - AtmosphereProcessType type() const override { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type() const override { + return AtmosphereProcessType::Physics; + } // The name of the subcomponent - std::string name() const override { return "mam_dry_deposition"; } + std::string name() const override { return "mam_dry_deposition"; } // grid void set_grids( @@ -87,9 +90,9 @@ class MAMDryDep final : public scream::AtmosphereProcess { void run_impl(const double dt) override; // Finalize - void finalize_impl() override {/*Do nothing*/}; + void finalize_impl() override{/*Do nothing*/}; - // Atmosphere processes often have a pre-processing step that constructs + // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. struct Preprocess { diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 8b110dc1634d..517ed4a53afd 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -36,6 +36,7 @@ initial_conditions: aerodynamical_resistance: 1e-5 w_updraft: 1e-5 pbl_height: 0.0 + phis: 0.1 # The parameters for I/O control Scorpio: From d0c6d2c873a35bc01aefe875fc0acfbe75ae975f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 06:56:59 -0700 Subject: [PATCH 130/477] Moves variables to match I/O --- ...x_mam_dry_deposition_process_interface.cpp | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index aa0f643b974a..6e492b8983a8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -158,6 +158,7 @@ void MAMDryDep::set_grids( // FIXME: Is is cldfrac_liq instead? find out add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + //----------- Variables from coupler (land component)--------- // Obukhov length [m] add_field("Obukhov_length", scalar2d, m, grid_name); @@ -167,17 +168,27 @@ void MAMDryDep::set_grids( // Land fraction [fraction] add_field("land_fraction", scalar2d, nondim, grid_name); + // Friction velocity from land model [m/s] + add_field("friction_velocity", scalar2d, m / s, grid_name); + + // Aerodynamical resistance from land model [s/m] + add_field("aerodynamical_resistance", scalar2d, s / m, grid_name); + + //----------- Variables from coupler (ice component)--------- + // Ice fraction [unitless] add_field("ice_fraction", scalar2d, nondim, grid_name); + //----------- Variables from coupler (ocean component)--------- // Ocean fraction [unitless] add_field("ocean_fraction", scalar2d, nondim, grid_name); - // Friction velocity from land model [m/s] - add_field("friction_velocity", scalar2d, m / s, grid_name); + // --------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // --------------------------------------------------------------------- - // Aerodynamical resistance from land model [s/m] - add_field("aerodynamical_resistance", scalar2d, s / m, grid_name); + add_field("wetdens", scalar4d_mid, kg / m3, grid_name); + add_field("dgncur_awet", scalar4d_mid, m, grid_name); // Tempary memory to format the wet_aero.int_aero_nmr and // wet_aero.int_aero_mmr views into the order expected by mam4xx. @@ -193,9 +204,6 @@ void MAMDryDep::set_grids( add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, grid_name); - add_field("wetdens", scalar4d_mid, kg / m3, grid_name); - add_field("dgncur_awet", scalar4d_mid, m, grid_name); - // TODO: The following are not used by drydep but to create a dry atmosphere // object. From 880ce7d94a9a4a52a63e99ee3ce801a0386f5ab8 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 06:59:23 -0700 Subject: [PATCH 131/477] Moves Updated vars above computed --- ...x_mam_dry_deposition_process_interface.cpp | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 6e492b8983a8..b56336f96cb8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -190,23 +190,6 @@ void MAMDryDep::set_grids( add_field("wetdens", scalar4d_mid, kg / m3, grid_name); add_field("dgncur_awet", scalar4d_mid, m, grid_name); - // Tempary memory to format the wet_aero.int_aero_nmr and - // wet_aero.int_aero_mmr views into the order expected by mam4xx. - add_field("qtracers", scalar4d_q, kg / kg, grid_name); - add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); - - // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, - 1 / (m * m) / s, grid_name); - // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, - 1 / (m * m) / s, grid_name); - add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, - grid_name); - - // TODO: The following are not used by drydep but to create a dry atmosphere - // object. - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for(int m = 0; m < num_aero_modes; ++m) { @@ -245,6 +228,20 @@ void MAMDryDep::set_grids( add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } + + // Tempary memory to format the wet_aero.int_aero_nmr and + // wet_aero.int_aero_mmr views into the order expected by mam4xx. + add_field("qtracers", scalar4d_q, kg / kg, grid_name); + add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); + + // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, + 1 / (m * m) / s, grid_name); + // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, + 1 / (m * m) / s, grid_name); + add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, + grid_name); } // ========================================================================================= From 1e76a7d0b58ec336449ab66722ea20a7f164d692 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 07:28:09 -0700 Subject: [PATCH 132/477] Some more i/o variable movements in initialize impl --- ...x_mam_dry_deposition_process_interface.cpp | 92 ++++++++++++------- .../eamxx/src/physics/mam/mam_coupling.hpp | 1 + 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index b56336f96cb8..6040a534fa05 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -229,9 +229,14 @@ void MAMDryDep::set_grids( "tracers"); } - // Tempary memory to format the wet_aero.int_aero_nmr and - // wet_aero.int_aero_mmr views into the order expected by mam4xx. + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- + add_field("qtracers", scalar4d_q, kg / kg, grid_name); + + // FIXME: what is the diff between d_qtracers_dt and tendencies???? + // FIXME: we might not need this in FM add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] @@ -240,21 +245,28 @@ void MAMDryDep::set_grids( // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, 1 / (m * m) / s, grid_name); + // FIXME: we might not need this in FM add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, grid_name); } -// ========================================================================================= -// ON HOST, returns the number of bytes of device memory needed by the above -// Buffer type given the number of columns and vertical levels +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by +// the above. Buffer type given the number of columns and vertical +// levels size_t MAMDryDep::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -// ========================================================================================= -// ON HOST, initializeŃ• the Buffer type with sufficient memory to store -// intermediate (dry) quantities on the given number of columns with the given -// number of vertical levels. Returns the number of bytes allocated. +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializeŃ• the Buffer type with sufficient memory to +// store intermediate (dry) quantities on the given number of +// columns with the given number of vertical levels. Returns the +// number of bytes allocated. void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -266,23 +278,30 @@ void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMDryDep."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMDryDep::initialize_impl(const RunType run_type) { - // Gather runtime options - //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + // Populate the wet atmosphere state with views from fields + // FIMXE: specifically look which among these are actually used by the process wet_atm_.qv = get_field_in("qv").get_view(); wet_atm_.qc = get_field_in("qc").get_view(); wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.cldfrac = get_field_in("cldfrac_tot") - .get_view(); // FIXME: tot or liq? - dry_atm_.pblh = get_field_in("pbl_height").get_view(); + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + // FIXME: tot or liq? + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); obukhov_length_ = get_field_in("Obukhov_length").get_view(); surface_friction_velocty_ = @@ -295,7 +314,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); + // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; dry_atm_.z_iface = buffer_.z_iface; dry_atm_.dz = buffer_.dz; @@ -307,19 +326,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.w_updraft = buffer_.w_updraft; dry_atm_.z_surf = 0.0; // FIXME: for now - dgncur_awet_ = get_field_out("dgncur_awet").get_view(); - wet_dens_ = get_field_out("wetdens").get_view(); - - qtracers_ = get_field_out("qtracers").get_view(); - d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); - - aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") - .get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") - .get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios + // ---- set wet/dry aerosol-related gas state data for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); @@ -360,9 +367,28 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } - // set up our preprocess functor + // ------------------------------------------------------------- + // Output fields for the process + // ------------------------------------------------------------- + + // FIXME: We might need to get rid of few of these fields + // as we do not need them in FM + dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); + qtracers_ = get_field_out("qtracers").get_view(); + d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + .get_view(); + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + .get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); + + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + // FIXME: Where is post processing functor???? } namespace { void compute_tendencies( diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 2bc76cec3311..189b78b044d2 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -610,6 +610,7 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); + //FIXME : add an assert statement to check id z_surf is zero or not PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output team.team_barrier(); // likely necessary to have z_iface up to date From 34a7099b040d7ee28f0c4f2c17580f5ffcfff572 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 11 Jul 2024 09:35:41 -0600 Subject: [PATCH 133/477] Use TeamVectorRange Also, only call kernel if one of iop_nudge_tq or iop_nudge_uv is true --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 37e6c9de8adc..74579df552bd 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -577,32 +577,34 @@ apply_iop_forcing(const Real dt) }); team.team_barrier(); - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { - if (iop_nudge_tq) { - // Restrict nudging of T and qv to certain levels if requested by user - // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high - // is in units of [hPa], thus convert iop_nudge_tq_low/high - Mask nudge_level(false); - int max_size = hyam.size(); - for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { - const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; - nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + if (iop_nudge_tq or iop_nudge_uv) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { + if (iop_nudge_tq) { + // Restrict nudging of T and qv to certain levels if requested by user + // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high + // is in units of [hPa], thus convert iop_nudge_tq_low/high + Mask nudge_level(false); + int max_size = hyam.size(); + for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { + const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; + nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 + and + pressure_from_iop >= iop_nudge_tq_high*100); + } + + qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); + temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); + + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); } - - qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); - temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - } - if (iop_nudge_uv) { - u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); - v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); - } - }); + if (iop_nudge_uv) { + u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); + v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); + } + }); + } // Release WS views ws.release_many_contiguous<1>({&pmid}); From 2443c697855c1bdc4ab695eae21f573ae3803070 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 09:46:27 -0700 Subject: [PATCH 134/477] Adds a new file and remove that code the the cpp file --- .../eamxx_mam_dry_deposition_functions.hpp | 99 +++++++++++++++++++ ...x_mam_dry_deposition_process_interface.cpp | 16 +-- 2 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp new file mode 100644 index 000000000000..f62d092523e4 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -0,0 +1,99 @@ +#ifndef EAMXX_MAM_DRY_DEPOSITION_FUNCTIONS_HPP +#define EAMXX_MAM_DRY_DEPOSITION_FUNCTIONS_HPP + +#include +#include + +namespace scream { + +namespace { +void compute_tendencies( + const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, + const double dt, const MAMDryDep::const_view_1d obklen, + const MAMDryDep::const_view_1d surfric, + const MAMDryDep::const_view_1d landfrac, + const MAMDryDep::const_view_1d icefrac, + const MAMDryDep::const_view_1d ocnfrac, + const MAMDryDep::const_view_1d friction_velocity, + const MAMDryDep::const_view_1d aerodynamical_resistance, + MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, + const mam_coupling::DryAtmosphere dry_atm, + const mam_coupling::AerosolState dry_aero, + const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies) { + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol, nlev); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { + const int num_aero_modes = mam_coupling::num_aero_modes(); + const int num_aero_species = mam_coupling::num_aero_species(); + const int icol = team.league_rank(); + const Real t = 0; + + compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); + team.team_barrier(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { + for(int mode = 0; mode < num_aero_modes; ++mode) { + int icnst = mam4::ConvProc::numptrcw_amode(mode); + qtracers(icol, lev, icnst) = + wet_aero.int_aero_nmr[mode](icol, lev); + for(int species = 0; species < num_aero_species; ++species) { + icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); + if(-1 < icnst) { + qtracers(icol, lev, icnst) = + wet_aero.int_aero_mmr[mode][species](icol, lev); + } + } + } + }); + team.team_barrier(); + + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + mam4::Surface surf; + mam4::Diagnostics diags; + mam4::Tendencies tends; + + for(int i = 0; i < num_aero_modes; ++i) { + diags.wet_geometric_mean_diameter_i[i] = + ekat::subview(dgncur_awet_, i, icol); + diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); + } + diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); + diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); + diags.deposition_flux_of_cloud_borne_aerosols = + ekat::subview(aerdepdrycw, icol); + diags.deposition_flux_of_interstitial_aerosols = + ekat::subview(aerdepdryis, icol); + + diags.Obukhov_length = obklen[icol]; + diags.surface_friction_velocty = surfric[icol]; + diags.land_fraction = landfrac[icol]; + diags.ice_fraction = icefrac[icol]; + diags.ocean_fraction = ocnfrac[icol]; + diags.friction_velocity = friction_velocity[icol]; + diags.aerodynamical_resistance = aerodynamical_resistance[icol]; + + // Fill Tendency views + for(int m = 0; m < num_aero_modes; ++m) { + int iconv = mam4::ConvProc::numptrcw_amode(m); + tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); + for(int a = 0; a < num_aero_species; ++a) { + iconv = mam4::ConvProc::lmassptrcw_amode(a, m); + if(-1 < iconv) + tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); + } + } + + const mam4::AeroConfig aero_config; + dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, + progs, diags, tends); + }); +} +} // namespace +} // namespace scream + +#endif \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 6040a534fa05..d2336d814f28 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -1,6 +1,9 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" -#include "mam4xx/drydep.hpp" +// ACI functions are stored in the following hpp file +#include + +#include "mam4xx/drydep.hpp"//FIXME: Do we need it here??? /* Inputs: @@ -241,10 +244,10 @@ void MAMDryDep::set_grids( // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, - 1 / (m * m) / s, grid_name); + 1 / m2 / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, - 1 / (m * m) / s, grid_name); + 1 / m2 / s, grid_name); // FIXME: we might not need this in FM add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, grid_name); @@ -379,9 +382,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -390,6 +393,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_aero_); // FIXME: Where is post processing functor???? } +/* namespace { void compute_tendencies( const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, @@ -478,7 +482,7 @@ void compute_tendencies( }); } } // namespace - +*/ // ========================================================================================= void MAMDryDep::run_impl(const double dt) { using DryDep = mam4::DryDeposition; From 518ab8061cf97d4f9893a27eb1b41ef3bc783c42 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 10:51:21 -0700 Subject: [PATCH 135/477] Removes the commented out code that is moved to a new file --- ...x_mam_dry_deposition_process_interface.cpp | 97 +------------------ 1 file changed, 4 insertions(+), 93 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index d2336d814f28..7fd56f490981 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -3,7 +3,7 @@ // ACI functions are stored in the following hpp file #include -#include "mam4xx/drydep.hpp"//FIXME: Do we need it here??? +#include "mam4xx/drydep.hpp" //FIXME: Do we need it here??? /* Inputs: @@ -382,9 +382,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); + tendencies_ = get_field_out("Tendencies").get_view(); //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -393,96 +393,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_aero_); // FIXME: Where is post processing functor???? } -/* -namespace { -void compute_tendencies( - const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, - const double dt, const MAMDryDep::const_view_1d obklen, - const MAMDryDep::const_view_1d surfric, - const MAMDryDep::const_view_1d landfrac, - const MAMDryDep::const_view_1d icefrac, - const MAMDryDep::const_view_1d ocnfrac, - const MAMDryDep::const_view_1d friction_velocity, - const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, - const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, - const mam_coupling::DryAtmosphere dry_atm, - const mam_coupling::AerosolState dry_aero, - const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies) { - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy( - ncol, nlev); - Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { - const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); - const int icol = team.league_rank(); - const Real t = 0; - - compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); - team.team_barrier(); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { - for(int mode = 0; mode < num_aero_modes; ++mode) { - int icnst = mam4::ConvProc::numptrcw_amode(mode); - qtracers(icol, lev, icnst) = - wet_aero.int_aero_nmr[mode](icol, lev); - for(int species = 0; species < num_aero_species; ++species) { - icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); - if(-1 < icnst) { - qtracers(icol, lev, icnst) = - wet_aero.int_aero_mmr[mode][species](icol, lev); - } - } - } - }); - team.team_barrier(); - - mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); - mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::Surface surf; - mam4::Diagnostics diags; - mam4::Tendencies tends; - - for(int i = 0; i < num_aero_modes; ++i) { - diags.wet_geometric_mean_diameter_i[i] = - ekat::subview(dgncur_awet_, i, icol); - diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); - } - diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); - diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - diags.deposition_flux_of_cloud_borne_aerosols = - ekat::subview(aerdepdrycw, icol); - diags.deposition_flux_of_interstitial_aerosols = - ekat::subview(aerdepdryis, icol); - - diags.Obukhov_length = obklen[icol]; - diags.surface_friction_velocty = surfric[icol]; - diags.land_fraction = landfrac[icol]; - diags.ice_fraction = icefrac[icol]; - diags.ocean_fraction = ocnfrac[icol]; - diags.friction_velocity = friction_velocity[icol]; - diags.aerodynamical_resistance = aerodynamical_resistance[icol]; - - // Fill Tendency views - for(int m = 0; m < num_aero_modes; ++m) { - int iconv = mam4::ConvProc::numptrcw_amode(m); - tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); - for(int a = 0; a < num_aero_species; ++a) { - iconv = mam4::ConvProc::lmassptrcw_amode(a, m); - if(-1 < iconv) - tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); - } - } - - const mam4::AeroConfig aero_config; - dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, - progs, diags, tends); - }); -} -} // namespace -*/ + // ========================================================================================= void MAMDryDep::run_impl(const double dt) { using DryDep = mam4::DryDeposition; From 86954a8b3df9fa73bea85031e7e924d514d0fb78 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 12:14:07 -0700 Subject: [PATCH 136/477] Removes qtracer from the output and allocate it locally --- .../mam/eamxx_mam_dry_deposition_process_interface.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 7fd56f490981..1a600bd34921 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -236,8 +236,6 @@ void MAMDryDep::set_grids( // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- - add_field("qtracers", scalar4d_q, kg / kg, grid_name); - // FIXME: what is the diff between d_qtracers_dt and tendencies???? // FIXME: we might not need this in FM add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); @@ -378,7 +376,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // as we do not need them in FM dgncur_awet_ = get_field_out("dgncur_awet").get_view(); wet_dens_ = get_field_out("wetdens").get_view(); - qtracers_ = get_field_out("qtracers").get_view(); d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); @@ -386,6 +383,12 @@ void MAMDryDep::initialize_impl(const RunType run_type) { .get_view(); tendencies_ = get_field_out("Tendencies").get_view(); + //----------------------------------------------------------------- + // Allocate memory + //----------------------------------------------------------------- + const int pcnst = mam4::aero_model::pcnst; + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- From bd91719612f63033b308e5eaba63e05768f1c3d6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Jul 2024 12:28:29 -0700 Subject: [PATCH 137/477] Removes d_qtracers_dt_ as output and allocate it as a local var --- ...x_mam_dry_deposition_process_interface.cpp | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 1a600bd34921..1c5527820736 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -249,7 +249,7 @@ void MAMDryDep::set_grids( // FIXME: we might not need this in FM add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, grid_name); -} +} // set_grids // ================================================================ // REQUEST_BUFFER_SIZE_IN_BYTES @@ -259,7 +259,7 @@ void MAMDryDep::set_grids( // levels size_t MAMDryDep::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); -} +} // requested_buffer_size_in_bytes // ================================================================ // INIT_BUFFERS @@ -277,7 +277,7 @@ void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMDryDep."); -} +} // init_buffers // ================================================================ // INITIALIZE_IMPL @@ -374,10 +374,10 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: We might need to get rid of few of these fields // as we do not need them in FM - dgncur_awet_ = get_field_out("dgncur_awet").get_view(); - wet_dens_ = get_field_out("wetdens").get_view(); - d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); - aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); + // d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); + aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); @@ -387,7 +387,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Allocate memory //----------------------------------------------------------------- const int pcnst = mam4::aero_model::pcnst; - qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + // FIXME: comment what they are and units..... + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + d_qtracers_dt_ = view_3d("d_qtracers_dt", ncol_, nlev_, pcnst); //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -395,7 +397,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); // FIXME: Where is post processing functor???? -} +} // initialize_impl // ========================================================================================= void MAMDryDep::run_impl(const double dt) { @@ -437,6 +439,5 @@ void MAMDryDep::run_impl(const double dt) { // Outputs: aerdepdrycw_, aerdepdryis_, tendencies_); Kokkos::fence(); -} -// ========================================================================================= +} // run_impl } // namespace scream From de33553985085fc53324d87fb31825abee7fbe80 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 11 Jul 2024 12:45:16 -0700 Subject: [PATCH 138/477] read in IOP lev data before surface pressure adjustment is applied --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 657f7ad742a0..70f1c41b603a 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -541,6 +541,15 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); + // Read in IOP lev data + auto data = iop_file_pressure.get_view().data(); + scorpio::read_var(iop_file,"lev",data); + + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilev(); From b019f2e2f5b1c2343786ae7fcd138d45638b4539 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Thu, 11 Jul 2024 15:42:50 -0700 Subject: [PATCH 139/477] Fix bug in io with registering avg_cnt vars in Append mode. This commit addresses a bug where average count tracking variables may be registered again when a file is in Append mode. There is logic in the registration of typical output variables that was missing for average count variables. Addresses #2898 --- .../eamxx/src/share/io/scorpio_output.cpp | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 4cb3f50403ac..a105e244d76f 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,11 +1038,42 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { + std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - scorpio::define_var(filename, name, "unitless", vec_of_dims, - "real",fp_precision, m_add_time_dim); + if (mode==scorpio::FileMode::Append) { + // Similar to the regular fields above, check that the var is in the file, and has the right properties + EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), + "Error! Cannot append, due to variable missing from the file.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n"); + const auto& var = scorpio::get_var(filename,name); + EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, + "Error! Cannot append, due to variable dimensions mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var dims : " + ekat::join(vec_of_dims,",") + "\n" + " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + EKAT_REQUIRE_MSG (var.units==unitless, + "Error! Cannot append, due to variable units mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var units: " + unitless + "\n" + " - var units from file: " + var.units + "\n"); + EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, + "Error! Cannot append, due to time dependency mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" + " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); + } else { + // Note, unlike with regular output variables, for the average counting + // variables we don't need to add all of the extra metadata. So we simply + // define the variable. + scorpio::define_var(filename, name, unitless, vec_of_dims, + "real",fp_precision, m_add_time_dim); + } } } } // register_variables From 0f86fe3c757f4abcb9e3ccbe6cab97258e5e5502 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 11 Jul 2024 16:21:55 -0700 Subject: [PATCH 140/477] add minimum threshold when checking lat and lon relative error for IOP file to prevent NaN --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 70f1c41b603a..42ea7d0ec03f 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - m_params.get("target_latitude"); + ekat::impl::max(m_params.get("target_latitude"),0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - m_params.get("target_longitude"); + ekat::impl::max(m_params.get("target_longitude"),0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), From 2f2319675b4aae88a6e3b873090760ee4216b1b3 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Fri, 12 Jul 2024 02:43:08 -0700 Subject: [PATCH 141/477] add a general DPxx compset and a DPxx RCE compset --- .../eamxx/cime_config/config_component.xml | 12 +++++++ .../eamxx/cime_config/config_compsets.xml | 18 ++++++++-- .../cime_config/namelist_defaults_scream.xml | 8 ++--- .../usermods_dirs/rcemip/user_nl_cpl | 34 +++++++++++++++++++ 4 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl diff --git a/components/eamxx/cime_config/config_component.xml b/components/eamxx/cime_config/config_component.xml index 0aeccd1e6d24..d3b6e34744d0 100644 --- a/components/eamxx/cime_config/config_component.xml +++ b/components/eamxx/cime_config/config_component.xml @@ -60,6 +60,18 @@ + + + char + + + + $SRCROOT/components/eamxx/cime_config/usermods_dirs/rcemip + + run_component_cam + env_case.xml + User mods to apply to specific compset matches. + scream default diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index 6b7035e9c813..d33b865bde93 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -80,6 +80,18 @@ Experimental, under development + + F2010-SCREAMv1-DP + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx + Experimental, under development + + + + F2000-SCREAMv1-RCE-DP + 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx + Experimental, under development + + @@ -93,19 +105,19 @@ - 864 + 864 - TRUE + TRUE - TRUE + TRUE diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50188236e4e0..36b72df87ff1 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -363,7 +363,7 @@ be lost if SCREAM_HACK_XML is not enabled. 3 3 3 - 3 + 3 4 true false @@ -597,10 +597,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0 - 2 + 2 False 2 - 1 + 1 1 6 0 @@ -614,7 +614,7 @@ be lost if SCREAM_HACK_XML is not enabled. 1 1 3.4e-08 - 0.216784 + 0.216784 UNSET 250000.0 250000.0 diff --git a/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl new file mode 100644 index 000000000000..3ecd465f7a14 --- /dev/null +++ b/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -0,0 +1,34 @@ +!------------------------------------------------------------------------ +! Users should ONLY USE user_nl_cpl to change namelists variables +! for namelist variables in drv_in (except for the ones below) and +! any keyword/values in seq_maps.rc +! Users should add ALL user specific namelist and seq_maps.rc changes below +! using the following syntax +! namelist_var = new_namelist_value +! or +! mapname = new_map_name +! For example to change the default value of ocn2atm_fmapname to 'foo' use +! ocn2atm_fmapname = 'foo' +! +! Note that some namelist variables MAY NOT be changed in user_nl_cpl - +! they are defined in a $CASEROOT xml file and must be changed with +! xmlchange. +! +! For example, rather than set username to 'foo' in user_nl_cpl, call +! ./xmlchange USER=foo +!------------------------------------------------------------------------ +! +! SPECIAL NOTE FOR AQUAPLANET +! Do not modify any of the following orb_ entries +! co2vmr is not specified via the namelist but rather in the env_run.xml +! +orb_eccen = 0 +orb_obliq = 0 +orb_mvelp = 0 +orb_mode = "fixed_parameters" + +seq_flux_mct_albdif = 0.07 +seq_flux_mct_albdir = 0.07 +seq_flux_atmocn_minwind = 1 + +constant_zenith_deg = 42.05 \ No newline at end of file From 66b1a7d768cd833344c3ac048c25c565f39e4b88 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Fri, 12 Jul 2024 09:31:44 -0700 Subject: [PATCH 142/477] add default gas emissions for RCE --- components/eamxx/cime_config/namelist_defaults_scream.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 36b72df87ff1..bc09ce40952d 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -343,6 +343,11 @@ be lost if SCREAM_HACK_XML is not enabled. 531.2820e-12 0.7906 1.0e-7 + 1650.e-9 + 348.0e-6 + 306.0e-9 + 0.0 + 0.0 From b00a709d4a9da196e40c19dac0349854521cc70a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 10:37:48 -0700 Subject: [PATCH 143/477] Adds missing call to update the state (int and cldbrn aerosols) --- .../physics/mam/eamxx_mam_wetscav_process_interface.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c6153794257a..c62dcd073b09 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -391,8 +391,8 @@ void MAMWetscav::run_impl(const double dt) { //----------- Variables from convective scheme ------------- - // TODO: Following variables are from convective parameterization (not implemented - // yet in EAMxx), so should be zero for now + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now auto sh_frac = view_2d("sh_frac", ncol_, nlev_); Kokkos::deep_copy(sh_frac, 0); @@ -527,6 +527,11 @@ void MAMWetscav::run_impl(const double dt) { } }); }); // icol parallel_for loop + + // call post processing to convert dry mixing ratios to wet mixing ratios + // and update the state + Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + Kokkos::fence(); // wait before returning to calling function } // ========================================================================================= From 4734cac912b4f51fb25f04957400299df8c47187 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 11:30:04 -0700 Subject: [PATCH 144/477] Updates number mixing ratios for state --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index c62dcd073b09..4f1af79a057b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -519,6 +519,9 @@ void MAMWetscav::run_impl(const double dt) { // update interstitial aerosol state Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const auto n_mode_i = progs.n_mode_i[m]; + const auto tends_n_mode_i = tends.n_mode_i[m]; + n_mode_i(kk) += tends_n_mode_i(kk) * dt; for(int a = 0; a < mam4::num_species_mode(m); ++a) { const auto q_aero_i = progs.q_aero_i[m][a]; const auto tends_q_aero_i = tends.q_aero_i[m][a]; From e69425bba8efa25131e6a86cf7e0b8e196aba00d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 16:22:05 -0700 Subject: [PATCH 145/477] Commented out code with some new variables-code compiles fine --- .../eamxx_mam_dry_deposition_functions.hpp | 38 ++++++++++++- ...x_mam_dry_deposition_process_interface.cpp | 55 +++++++++++-------- ...x_mam_dry_deposition_process_interface.hpp | 4 ++ .../single-process/mam/drydep/input.yaml | 1 + 4 files changed, 72 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index f62d092523e4..17e6885c842d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -2,6 +2,8 @@ #define EAMXX_MAM_DRY_DEPOSITION_FUNCTIONS_HPP #include +#include +#include #include namespace scream { @@ -17,11 +19,13 @@ void compute_tendencies( const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, + const MAMDryDep::const_view_2d fraction_landuse, const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies) { + MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies, + MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst]) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( ncol, nlev); @@ -89,8 +93,36 @@ void compute_tendencies( } const mam4::AeroConfig aero_config; - dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, - progs, diags, tends); + /*dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, + progs, diags, tends);*/ + mam4::ColumnView qqcw_tends[mam4::aero_model::pcnst]; + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { + qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); + } + // Extract Prognostics + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + for(int m = 0; m < mam4::AeroConfig::num_modes(); ++m) { + // qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = + // progs.n_mode_c[m][kk]; + qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = + progs.n_mode_c[m][kk]; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + qqcw_tends[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = + progs.q_aero_c[m][a][kk]; + } + }); + /*mam4::aero_model_drydep( + team, fraction_landuse, atm.temperature, atm.pressure, + atm.interface_pressure , atm.hydrostatic_dp, ekat::subview(qtracers, + icol), ekat::subview(dgncur_awet_, i, icol), ekat::subview(wet_dens_, + i, icol), qqcw_tends, obklen[icol], surfric[icol], landfrac[icol], + icefrac[icol], ocnfrac[icol], friction_velocity[icol], + aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), + ptend_lq, dt, ekat::subview(aerdepdrycw, icol), + ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, + dqdt_tmp);*/ }); } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 1c5527820736..ae6b0819c10f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -102,6 +102,16 @@ void MAMDryDep::set_grids( FieldLayout scalar4d_q{{COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; FieldLayout scalar4d_qqcw_tends{{COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; + auto make_layout = [](const std::vector &extents, + const std::vector &names) { + std::vector tags(extents.size(), CMP); + return FieldLayout(tags, extents, names); + }; + + static constexpr int n_land_type = 11; + FieldLayout scalar3d_landtype = + make_layout({ncol_, n_land_type}, {"COL", "N_LAND_TYPE"}); + using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers @@ -186,12 +196,19 @@ void MAMDryDep::set_grids( // Ocean fraction [unitless] add_field("ocean_fraction", scalar2d, nondim, grid_name); + //----------- Variables from other mam4xx processes ------------ + // geometric mean wet diameter for number distribution [m] + add_field("dgncur_awet", scalar4d_mid, m, grid_name); + + //----------- FIXME:Variables to revisit------------------------------- + // FIXME:[TEMPORARY] These variables should come from data files + add_field("fraction_landuse", scalar4d_mid, m, grid_name); + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- add_field("wetdens", scalar4d_mid, kg / m3, grid_name); - add_field("dgncur_awet", scalar4d_mid, m, grid_name); // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -305,15 +322,17 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.pblh = get_field_in("pbl_height").get_view(); obukhov_length_ = get_field_in("Obukhov_length").get_view(); - surface_friction_velocty_ = - get_field_in("surface_friction_velocty").get_view(); land_fraction_ = get_field_in("land_fraction").get_view(); ice_fraction_ = get_field_in("ice_fraction").get_view(); ocean_fraction_ = get_field_in("ocean_fraction").get_view(); + // geometric mean wet diameter for number distribution [m] + dgncur_awet_ = get_field_out("dgncur_awet").get_view(); friction_velocity_ = get_field_in("friction_velocity").get_view(); aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); + surface_friction_velocty_ = + get_field_in("surface_friction_velocty").get_view(); // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; @@ -374,9 +393,8 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: We might need to get rid of few of these fields // as we do not need them in FM - dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + wet_dens_ = get_field_out("wetdens").get_view(); - // d_qtracers_dt_ = get_field_out("d_qtracers_dt").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") @@ -389,7 +407,10 @@ void MAMDryDep::initialize_impl(const RunType run_type) { const int pcnst = mam4::aero_model::pcnst; // FIXME: comment what they are and units..... qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - d_qtracers_dt_ = view_3d("d_qtracers_dt", ncol_, nlev_, pcnst); + d_qtracers_dt_ = view_3d("d_qtracers_d_t", ncol_, nlev_, pcnst); + for(int i = 0; i < pcnst; ++i) { + Kokkos::resize(qqcw_tends_[i], ncol_, nlev_); + } //----------------------------------------------------------------- // Setup preprocessing and post processing @@ -410,34 +431,22 @@ void MAMDryDep::run_impl(const double dt) { Kokkos::fence(); const DryDep::Config process_config; - // Future enhancement: - // This is where we can set the fraction of land use parameters: - // if (do_soilw) { - // const Real *landuse = get_landuse_from_file(do_soilw); - // for (int i=0; i::get_default_team_policy(ncol_, nlev_); - // Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - // dry_deposition.validate( aero_config, team, atm, progs); - // } - // Kokkos::fence(); + auto fraction_landuse = + get_field_in("fraction_landuse").get_view(); compute_tendencies(ncol_, nlev_, dry_deposition, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, - aerodynamical_resistance_, qtracers_, - d_qtracers_dt_, // d_qtracers_dt_ is an output + aerodynamical_resistance_, qtracers_, d_qtracers_dt_, + fraction_landuse, // d_qtracers_dt_ is an output dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, // Outputs: - aerdepdrycw_, aerdepdryis_, tendencies_); + aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_); Kokkos::fence(); } // run_impl } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 1144bd79ff68..ae764e0baf5a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -42,12 +42,16 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_3d qtracers_; view_3d d_qtracers_dt_; + + // inputs + // FIXME: collect all inputs and outputs together view_3d dgncur_awet_; view_3d wet_dens_; view_3d tendencies_; view_2d aerdepdrycw_; view_2d aerdepdryis_; + view_2d qqcw_tends_[mam4::aero_model::pcnst]; const_view_1d obukhov_length_; const_view_1d surface_friction_velocty_; diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 517ed4a53afd..5ebfbe4acfbc 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -34,6 +34,7 @@ initial_conditions: ocean_fraction: 0.0 friction_velocity: 1e-5 aerodynamical_resistance: 1e-5 + fraction_landuse: 0.2 w_updraft: 1e-5 pbl_height: 0.0 phis: 0.1 From c1bbdf5598927f8d3072847f99b30a2f9b69166f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 19:23:52 -0700 Subject: [PATCH 146/477] Adds new vars and allocate --- .../eamxx_mam_dry_deposition_process_interface.cpp | 12 ++++++++++++ .../eamxx_mam_dry_deposition_process_interface.hpp | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index ae6b0819c10f..2eb42f668264 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -408,8 +408,20 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: comment what they are and units..... qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); d_qtracers_dt_ = view_3d("d_qtracers_d_t", ncol_, nlev_, pcnst); + + rho_ = view_2d("rho", ncol_, nlev_); + + for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { + for(int j = 0; j < aerosol_categories_; ++j) { + Kokkos::resize(vlc_dry_[i][j], ncol_, nlev_); + Kokkos::resize(vlc_grv_[i][j], ncol_, nlev_); + } + } + + vlc_dry_, vlc_trb_, vlc_grv_; for(int i = 0; i < pcnst; ++i) { Kokkos::resize(qqcw_tends_[i], ncol_, nlev_); + Kokkos::resize(dqdt_tmp_[i], ncol_, nlev_); } //----------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index ae764e0baf5a..08dcf4dfe3c1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -47,6 +47,12 @@ class MAMDryDep final : public scream::AtmosphereProcess { // FIXME: collect all inputs and outputs together view_3d dgncur_awet_; view_3d wet_dens_; + view_2d rho_; + static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx + view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; + view_2d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; + view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; + view_2d dqdt_tmp_[mam4::aero_model::pcnst]; view_3d tendencies_; view_2d aerdepdrycw_; From 0532e27701342a45bdfb092f5a73c9524fa00c7c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Jul 2024 19:41:25 -0700 Subject: [PATCH 147/477] added as args --- .../src/physics/mam/eamxx_mam_dry_deposition_functions.hpp | 7 ++++++- .../mam/eamxx_mam_dry_deposition_process_interface.cpp | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 17e6885c842d..e328cc4036c0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -25,7 +25,12 @@ void compute_tendencies( const mam_coupling::AerosolState dry_aero, const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies, - MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst]) { + MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst], + MAMDryDep::view_2d rho_, + MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()][4], + MAMDryDep::view_2d vlc_trb_[mam4::AeroConfig::num_modes()][4], + MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()][4], + MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( ncol, nlev); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 2eb42f668264..004a5eab08f9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -458,7 +458,8 @@ void MAMDryDep::run_impl(const double dt) { fraction_landuse, // d_qtracers_dt_ is an output dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, // Outputs: - aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_); + aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_, rho_, + vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); } // run_impl } // namespace scream From 33202bec2246a1340cfc5b75c765ec5280f5c381 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 10:05:48 -0700 Subject: [PATCH 148/477] Submodule change, commented out code but compiles --- .../eamxx_mam_dry_deposition_functions.hpp | 33 +++++++++++++------ ...x_mam_dry_deposition_process_interface.cpp | 25 +++++++------- ...x_mam_dry_deposition_process_interface.hpp | 3 +- externals/mam4xx | 2 +- 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index e328cc4036c0..58a263f723e8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,7 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, - const MAMDryDep::const_view_2d fraction_landuse, + MAMDryDep::view_1d fraction_landuse_[mam4::DryDeposition::n_land_type], const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, @@ -65,11 +65,14 @@ void compute_tendencies( mam4::Surface surf; mam4::Diagnostics diags; mam4::Tendencies tends; + mam4::ColumnView dgncur_awet[num_aero_modes], wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { diags.wet_geometric_mean_diameter_i[i] = ekat::subview(dgncur_awet_, i, icol); + dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); + wet_dens[i] = ekat::subview(wet_dens_, i, icol); } diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); @@ -100,6 +103,15 @@ void compute_tendencies( const mam4::AeroConfig aero_config; /*dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, progs, diags, tends);*/ + + mam4::ColumnView rho; + rho = ekat::subview(rho_, icol); + + Real fraction_landuse[mam4::DryDeposition::n_land_type]; + for(int i = 0; i < mam4::DryDeposition::n_land_type; ++i) { + fraction_landuse[i] = fraction_landuse_[i](icol); + } + mam4::ColumnView qqcw_tends[mam4::aero_model::pcnst]; for(int i = 0; i < mam4::aero_model::pcnst; ++i) { qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); @@ -118,16 +130,17 @@ void compute_tendencies( progs.q_aero_c[m][a][kk]; } }); - /*mam4::aero_model_drydep( + bool ptend_lq[mam4::aero_model::pcnst]; + mam4::aero_model_drydep( team, fraction_landuse, atm.temperature, atm.pressure, - atm.interface_pressure , atm.hydrostatic_dp, ekat::subview(qtracers, - icol), ekat::subview(dgncur_awet_, i, icol), ekat::subview(wet_dens_, - i, icol), qqcw_tends, obklen[icol], surfric[icol], landfrac[icol], - icefrac[icol], ocnfrac[icol], friction_velocity[icol], - aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), - ptend_lq, dt, ekat::subview(aerdepdrycw, icol), - ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, - dqdt_tmp);*/ + atm.interface_pressure, atm.hydrostatic_dp, + ekat::subview(qtracers, icol), dgncur_awet, wet_dens, qqcw_tends, + obklen[icol], surfric[icol], landfrac[icol], icefrac[icol], + ocnfrac[icol], friction_velocity[icol], + aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), + ptend_lq, dt, ekat::subview(aerdepdrycw, icol), + ekat::subview(aerdepdryis, icol)); /*, rho, vlc_dry, vlc_trb, + vlc_grv, dqdt_tmp);*/ }); } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 004a5eab08f9..1a3d0850f182 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -108,10 +108,6 @@ void MAMDryDep::set_grids( return FieldLayout(tags, extents, names); }; - static constexpr int n_land_type = 11; - FieldLayout scalar3d_landtype = - make_layout({ncol_, n_land_type}, {"COL", "N_LAND_TYPE"}); - using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers @@ -200,10 +196,6 @@ void MAMDryDep::set_grids( // geometric mean wet diameter for number distribution [m] add_field("dgncur_awet", scalar4d_mid, m, grid_name); - //----------- FIXME:Variables to revisit------------------------------- - // FIXME:[TEMPORARY] These variables should come from data files - add_field("fraction_landuse", scalar4d_mid, m, grid_name); - // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- @@ -325,8 +317,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { land_fraction_ = get_field_in("land_fraction").get_view(); ice_fraction_ = get_field_in("ice_fraction").get_view(); ocean_fraction_ = get_field_in("ocean_fraction").get_view(); - // geometric mean wet diameter for number distribution [m] - dgncur_awet_ = get_field_out("dgncur_awet").get_view(); friction_velocity_ = get_field_in("friction_velocity").get_view(); aerodynamical_resistance_ = @@ -418,12 +408,16 @@ void MAMDryDep::initialize_impl(const RunType run_type) { } } - vlc_dry_, vlc_trb_, vlc_grv_; for(int i = 0; i < pcnst; ++i) { Kokkos::resize(qqcw_tends_[i], ncol_, nlev_); Kokkos::resize(dqdt_tmp_[i], ncol_, nlev_); } + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; + for(int i = 0; i < n_land_type; ++i) { + Kokkos::resize(fraction_landuse_[i], ncol_); + } + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- @@ -448,14 +442,17 @@ void MAMDryDep::run_impl(const double dt) { const mam4::AeroConfig aero_config; dry_deposition.init(aero_config, process_config); - auto fraction_landuse = - get_field_in("fraction_landuse").get_view(); + // FIXME: There are some vars that are not declared "REQUIRED" etc. but still + // used!!! + + // geometric mean wet diameter for number distribution [m] + auto dgncur_awet_ = get_field_out("dgncur_awet").get_view(); compute_tendencies(ncol_, nlev_, dry_deposition, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, aerodynamical_resistance_, qtracers_, d_qtracers_dt_, - fraction_landuse, // d_qtracers_dt_ is an output + fraction_landuse_, // d_qtracers_dt_ is an output dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, // Outputs: aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_, rho_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 08dcf4dfe3c1..c423a20ebc89 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -16,6 +16,7 @@ namespace scream { // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { public: + using view_1d = Field::view_dev_t; using view_2d = Field::view_dev_t; using view_3d = Field::view_dev_t; using const_view_1d = Field::view_dev_t; @@ -45,7 +46,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { // inputs // FIXME: collect all inputs and outputs together - view_3d dgncur_awet_; + view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; view_3d wet_dens_; view_2d rho_; static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx diff --git a/externals/mam4xx b/externals/mam4xx index a8aa56ba915c..eb2d728412b4 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e +Subproject commit eb2d728412b43081afbd579230450da352e716fb From fba5bd77e7677a2b692d078aaf67d2574fd54711 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 13 Jul 2024 13:32:02 -0400 Subject: [PATCH 149/477] fix warning due to format type --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 9fb5726084f4..c406c1ebd9b4 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } From 309f2d28ef770610f4054f1ebe7697d59032f47f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 11:30:24 -0700 Subject: [PATCH 150/477] subm changes, All func args are in, compiles fine --- .../eamxx_mam_dry_deposition_functions.hpp | 36 +++++++++++++++---- ...x_mam_dry_deposition_process_interface.cpp | 1 + ...x_mam_dry_deposition_process_interface.hpp | 17 ++++----- externals/mam4xx | 2 +- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 58a263f723e8..806c98cdf128 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -27,9 +27,12 @@ void compute_tendencies( MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies, MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst], MAMDryDep::view_2d rho_, - MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()][4], - MAMDryDep::view_2d vlc_trb_[mam4::AeroConfig::num_modes()][4], - MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()][4], + MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], + MAMDryDep::view_2d vlc_trb_[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], + MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( @@ -112,9 +115,30 @@ void compute_tendencies( fraction_landuse[i] = fraction_landuse_[i](icol); } + // FIXME: why mam4::ColumnView didn;t work here, why use + // Kokkos::View. Solution: Use ColumnView in drydep.hpp as well. + Kokkos::View vlc_dry[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], + vlc_trb[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_], + vlc_grv[mam4::AeroConfig::num_modes()] + [MAMDryDep::aerosol_categories_]; + + for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { + for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { + vlc_dry[i][j] = ekat::subview(vlc_dry_[i][j], icol); + vlc_trb[i][j] = ekat::subview(vlc_trb_[i][j], icol); + vlc_grv[i][j] = ekat::subview(vlc_grv_[i][j], icol); + } + } + mam4::ColumnView qqcw_tends[mam4::aero_model::pcnst]; + Kokkos::View dqdt_tmp[mam4::aero_model::pcnst]; for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); + qqcw_tends[i] = ekat::subview( + qqcw_tends_[i], icol); // FIXME: Do we need qqcw_tends_, why + // can't we just use qqcw_tends + dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); } // Extract Prognostics Kokkos::parallel_for( @@ -139,8 +163,8 @@ void compute_tendencies( ocnfrac[icol], friction_velocity[icol], aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), ptend_lq, dt, ekat::subview(aerdepdrycw, icol), - ekat::subview(aerdepdryis, icol)); /*, rho, vlc_dry, vlc_trb, - vlc_grv, dqdt_tmp);*/ + ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, + dqdt_tmp); }); } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 1a3d0850f182..b19d32f41694 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -405,6 +405,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { for(int j = 0; j < aerosol_categories_; ++j) { Kokkos::resize(vlc_dry_[i][j], ncol_, nlev_); Kokkos::resize(vlc_grv_[i][j], ncol_, nlev_); + Kokkos::resize(vlc_trb_[i][j], ncol_, nlev_); } } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index c423a20ebc89..13d006005aa9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -16,13 +16,14 @@ namespace scream { // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { public: - using view_1d = Field::view_dev_t; - using view_2d = Field::view_dev_t; - using view_3d = Field::view_dev_t; - using const_view_1d = Field::view_dev_t; - using const_view_2d = Field::view_dev_t; - using const_view_3d = Field::view_dev_t; - static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + using view_1d = Field::view_dev_t; + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx private: // number of horizontal columns and vertical levels @@ -49,7 +50,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; view_3d wet_dens_; view_2d rho_; - static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx + view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; diff --git a/externals/mam4xx b/externals/mam4xx index eb2d728412b4..df7c3c451ccc 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit eb2d728412b43081afbd579230450da352e716fb +Subproject commit df7c3c451ccc4c0c6f7f2c779cac6ba8b598e059 From 8dca5fab793d6735f75fb5147c1bd81c1fb0552a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 12:40:59 -0700 Subject: [PATCH 151/477] Subm change; a working aero_model_drydep call --- .../eamxx_mam_dry_deposition_functions.hpp | 50 +++++++++---------- ...x_mam_dry_deposition_process_interface.cpp | 2 + ...x_mam_dry_deposition_process_interface.hpp | 42 ++++++++++++++++ .../eamxx/src/physics/mam/mam_coupling.hpp | 6 ++- externals/mam4xx | 2 +- 5 files changed, 74 insertions(+), 28 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 806c98cdf128..4ee76a54470b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -44,8 +44,6 @@ void compute_tendencies( const int icol = team.league_rank(); const Real t = 0; - compute_wet_mixing_ratios(team, dry_atm, dry_aero, wet_aero, icol); - team.team_barrier(); Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { for(int mode = 0; mode < num_aero_modes; ++mode) { @@ -65,35 +63,35 @@ void compute_tendencies( mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::Surface surf; - mam4::Diagnostics diags; - mam4::Tendencies tends; + // mam4::Surface surf; + // mam4::Diagnostics diags; + // mam4::Tendencies tends; mam4::ColumnView dgncur_awet[num_aero_modes], wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { - diags.wet_geometric_mean_diameter_i[i] = - ekat::subview(dgncur_awet_, i, icol); - dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); - diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); - wet_dens[i] = ekat::subview(wet_dens_, i, icol); + // diags.wet_geometric_mean_diameter_i[i] = + // ekat::subview(dgncur_awet_, i, icol); + dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); + // diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); + wet_dens[i] = ekat::subview(wet_dens_, i, icol); } - diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); - diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - diags.deposition_flux_of_cloud_borne_aerosols = - ekat::subview(aerdepdrycw, icol); - diags.deposition_flux_of_interstitial_aerosols = - ekat::subview(aerdepdryis, icol); + // diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); + // diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); + // diags.deposition_flux_of_cloud_borne_aerosols = + // ekat::subview(aerdepdrycw, icol); + // diags.deposition_flux_of_interstitial_aerosols = + // ekat::subview(aerdepdryis, icol); - diags.Obukhov_length = obklen[icol]; - diags.surface_friction_velocty = surfric[icol]; - diags.land_fraction = landfrac[icol]; - diags.ice_fraction = icefrac[icol]; - diags.ocean_fraction = ocnfrac[icol]; - diags.friction_velocity = friction_velocity[icol]; - diags.aerodynamical_resistance = aerodynamical_resistance[icol]; + // diags.Obukhov_length = obklen[icol]; + // diags.surface_friction_velocty = surfric[icol]; + // diags.land_fraction = landfrac[icol]; + // diags.ice_fraction = icefrac[icol]; + // diags.ocean_fraction = ocnfrac[icol]; + // diags.friction_velocity = friction_velocity[icol]; + // diags.aerodynamical_resistance = aerodynamical_resistance[icol]; // Fill Tendency views - for(int m = 0; m < num_aero_modes; ++m) { + /*for(int m = 0; m < num_aero_modes; ++m) { int iconv = mam4::ConvProc::numptrcw_amode(m); tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); for(int a = 0; a < num_aero_species; ++a) { @@ -101,9 +99,9 @@ void compute_tendencies( if(-1 < iconv) tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); } - } + }*/ - const mam4::AeroConfig aero_config; + // const mam4::AeroConfig aero_config; /*dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, progs, diags, tends);*/ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index b19d32f41694..ff29f71e89d8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -312,6 +312,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: tot or liq? dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); obukhov_length_ = get_field_in("Obukhov_length").get_view(); land_fraction_ = get_field_in("land_fraction").get_view(); @@ -459,5 +460,6 @@ void MAMDryDep::run_impl(const double dt) { aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_, rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); + // FIXME: Where is update tends and post processing???? } // run_impl } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 13d006005aa9..11ec5f044102 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -132,6 +132,10 @@ class MAMDryDep final : public scream::AtmosphereProcess { compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); } // operator() // local variables for preprocess struct @@ -144,6 +148,44 @@ class MAMDryDep final : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess + // Atmosphere processes often have a post-processing step prepares output + // from this process for the Field Manager. This functor implements this + // step, which is called during run_impl. + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_post_, nlev_post_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // Postprocess + private: // pre- and postprocessing scratch pads Preprocess preprocess_; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 189b78b044d2..030ffea88789 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -599,13 +599,16 @@ void compute_vertical_layer_heights(const Team& team, EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), "Given column index does not correspond to given team!"); + //outputs const auto dz = ekat::subview(dry_atm.dz, column_index); const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); + //inputs const auto qv = ekat::subview(dry_atm.qv, column_index); const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + // NOTE: we are using dry qv. Does calculate_dz require dry or wet? PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output @@ -614,7 +617,8 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output team.team_barrier(); // likely necessary to have z_iface up to date - PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); + PF::calculate_z_mid(team, mam4::nlev, z_iface, //input + z_mid); //output } // Given a thread team and wet and dry atmospheres, dispatches threads from the diff --git a/externals/mam4xx b/externals/mam4xx index df7c3c451ccc..1c8b5b6e0ff5 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit df7c3c451ccc4c0c6f7f2c779cac6ba8b598e059 +Subproject commit 1c8b5b6e0ff5ac20bc58907f3593f816762a77f2 From 17c09a0c78d415a7191a7095c071d8aa46ba7f10 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 12:47:11 -0700 Subject: [PATCH 152/477] Remove compute tends to drydep.hpp from drydep interface --- .../eamxx_mam_dry_deposition_functions.hpp | 45 ++----------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 4ee76a54470b..0204461f2bcd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -58,52 +58,17 @@ void compute_tendencies( } } } - }); + }); // parallel_for for nlevs team.team_barrier(); mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - // mam4::Surface surf; - // mam4::Diagnostics diags; - // mam4::Tendencies tends; mam4::ColumnView dgncur_awet[num_aero_modes], wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { - // diags.wet_geometric_mean_diameter_i[i] = - // ekat::subview(dgncur_awet_, i, icol); dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); - // diags.wet_density[i] = ekat::subview(wet_dens_, i, icol); - wet_dens[i] = ekat::subview(wet_dens_, i, icol); + wet_dens[i] = ekat::subview(wet_dens_, i, icol); } - // diags.tracer_mixing_ratio = ekat::subview(qtracers, icol); - // diags.d_tracer_mixing_ratio_dt = ekat::subview(d_qtracers_dt, icol); - // diags.deposition_flux_of_cloud_borne_aerosols = - // ekat::subview(aerdepdrycw, icol); - // diags.deposition_flux_of_interstitial_aerosols = - // ekat::subview(aerdepdryis, icol); - - // diags.Obukhov_length = obklen[icol]; - // diags.surface_friction_velocty = surfric[icol]; - // diags.land_fraction = landfrac[icol]; - // diags.ice_fraction = icefrac[icol]; - // diags.ocean_fraction = ocnfrac[icol]; - // diags.friction_velocity = friction_velocity[icol]; - // diags.aerodynamical_resistance = aerodynamical_resistance[icol]; - - // Fill Tendency views - /*for(int m = 0; m < num_aero_modes; ++m) { - int iconv = mam4::ConvProc::numptrcw_amode(m); - tends.n_mode_c[m] = ekat::subview(tendencies, icol, iconv); - for(int a = 0; a < num_aero_species; ++a) { - iconv = mam4::ConvProc::lmassptrcw_amode(a, m); - if(-1 < iconv) - tends.q_aero_c[m][a] = ekat::subview(tendencies, icol, iconv); - } - }*/ - - // const mam4::AeroConfig aero_config; - /*dry_deposition.compute_tendencies(aero_config, team, t, dt, atm, surf, - progs, diags, tends);*/ mam4::ColumnView rho; rho = ekat::subview(rho_, icol); @@ -142,8 +107,6 @@ void compute_tendencies( Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { for(int m = 0; m < mam4::AeroConfig::num_modes(); ++m) { - // qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = - // progs.n_mode_c[m][kk]; qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) @@ -151,7 +114,7 @@ void compute_tendencies( qqcw_tends[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = progs.q_aero_c[m][a][kk]; } - }); + }); // parallel_for nlevs bool ptend_lq[mam4::aero_model::pcnst]; mam4::aero_model_drydep( team, fraction_landuse, atm.temperature, atm.pressure, @@ -163,7 +126,7 @@ void compute_tendencies( ptend_lq, dt, ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); - }); + }); // parallel_for for ncols } } // namespace } // namespace scream From 1323ea6a4b5a1324f6b4a9d10299d6a2ae0d7e2a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 13:26:47 -0700 Subject: [PATCH 153/477] Zero out tends before computing tends as they had junk values --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 4f1af79a057b..91e69443b588 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -506,6 +506,14 @@ void MAMWetscav::run_impl(const double dt) { auto wetdens_icol = ekat::subview(wetdens, icol); const auto prain_icol = ekat::subview(prain, icol); + // Zero out tendencies otherwise, they are initialized to junk values + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(tends.n_mode_i[m], 0); + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + Kokkos::deep_copy(tends.q_aero_i[m][a], 0); + } + } + mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, // inputs @@ -528,8 +536,8 @@ void MAMWetscav::run_impl(const double dt) { q_aero_i(kk) += tends_q_aero_i(kk) * dt; } } - }); - }); // icol parallel_for loop + }); // parallel_for for update interstitial aerosol state + }); // icol parallel_for loop // call post processing to convert dry mixing ratios to wet mixing ratios // and update the state From 0dd177a071afec7b82a82d783c36efe40f2b172d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 13:29:57 -0700 Subject: [PATCH 154/477] Cleanup, subm changed, Use ColumnView for some views --- .../eamxx_mam_dry_deposition_functions.hpp | 50 +++++++++---------- ...x_mam_dry_deposition_process_interface.hpp | 14 +++--- externals/mam4xx | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 0204461f2bcd..0dc2caf63baa 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,7 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, - MAMDryDep::view_1d fraction_landuse_[mam4::DryDeposition::n_land_type], + MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, @@ -39,22 +39,24 @@ void compute_tendencies( ncol, nlev); Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { - const int num_aero_modes = mam_coupling::num_aero_modes(); - const int num_aero_species = mam_coupling::num_aero_species(); - const int icol = team.league_rank(); - const Real t = 0; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int num_aero_species = + mam_coupling::num_aero_species(); + static constexpr int n_land_type = MAMDryDep::n_land_type; + + const int icol = team.league_rank(); Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { for(int mode = 0; mode < num_aero_modes; ++mode) { int icnst = mam4::ConvProc::numptrcw_amode(mode); qtracers(icol, lev, icnst) = - wet_aero.int_aero_nmr[mode](icol, lev); + dry_aero.int_aero_nmr[mode](icol, lev); for(int species = 0; species < num_aero_species; ++species) { icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); if(-1 < icnst) { qtracers(icol, lev, icnst) = - wet_aero.int_aero_mmr[mode][species](icol, lev); + dry_aero.int_aero_mmr[mode][species](icol, lev); } } } @@ -73,40 +75,36 @@ void compute_tendencies( mam4::ColumnView rho; rho = ekat::subview(rho_, icol); - Real fraction_landuse[mam4::DryDeposition::n_land_type]; - for(int i = 0; i < mam4::DryDeposition::n_land_type; ++i) { + Real fraction_landuse[n_land_type]; + for(int i = 0; i < n_land_type; ++i) { fraction_landuse[i] = fraction_landuse_[i](icol); } // FIXME: why mam4::ColumnView didn;t work here, why use // Kokkos::View. Solution: Use ColumnView in drydep.hpp as well. - Kokkos::View vlc_dry[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - vlc_trb[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - vlc_grv[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_]; + static constexpr int nmodes = mam4::AeroConfig::num_modes(); + mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_], + vlc_trb[nmodes][MAMDryDep::aerosol_categories_], + vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; - for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { + for(int i = 0; i < nmodes; ++i) { for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { vlc_dry[i][j] = ekat::subview(vlc_dry_[i][j], icol); vlc_trb[i][j] = ekat::subview(vlc_trb_[i][j], icol); vlc_grv[i][j] = ekat::subview(vlc_grv_[i][j], icol); } } - - mam4::ColumnView qqcw_tends[mam4::aero_model::pcnst]; - Kokkos::View dqdt_tmp[mam4::aero_model::pcnst]; - for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - qqcw_tends[i] = ekat::subview( - qqcw_tends_[i], icol); // FIXME: Do we need qqcw_tends_, why - // can't we just use qqcw_tends - dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); + static constexpr int pcnst = mam4::aero_model::pcnst; + mam4::ColumnView qqcw_tends[pcnst]; + mam4::ColumnView dqdt_tmp[pcnst]; + for(int i = 0; i < pcnst; ++i) { + qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); + dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); } // Extract Prognostics Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { - for(int m = 0; m < mam4::AeroConfig::num_modes(); ++m) { + for(int m = 0; m < nmodes; ++m) { qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) @@ -115,7 +113,7 @@ void compute_tendencies( progs.q_aero_c[m][a][kk]; } }); // parallel_for nlevs - bool ptend_lq[mam4::aero_model::pcnst]; + bool ptend_lq[pcnst]; mam4::aero_model_drydep( team, fraction_landuse, atm.temperature, atm.pressure, atm.interface_pressure, atm.hydrostatic_dp, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 11ec5f044102..fdfb387524d8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -16,14 +16,16 @@ namespace scream { // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { public: - using view_1d = Field::view_dev_t; - using view_2d = Field::view_dev_t; - using view_3d = Field::view_dev_t; - using const_view_1d = Field::view_dev_t; - using const_view_2d = Field::view_dev_t; - using const_view_3d = Field::view_dev_t; + using view_1d = Field::view_dev_t; + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; private: // number of horizontal columns and vertical levels diff --git a/externals/mam4xx b/externals/mam4xx index 1c8b5b6e0ff5..d0ea0c4f68f4 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 1c8b5b6e0ff5ac20bc58907f3593f816762a77f2 +Subproject commit d0ea0c4f68f4a939ee73f92bdea797e597eafc3f From e9f52c0147e908f311ca6e8a07a680472367d75f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 15:40:34 -0700 Subject: [PATCH 155/477] Cleanup-makes dgn wet size a input and other cleanup --- .../eamxx_mam_dry_deposition_functions.hpp | 23 ++++++----- ...x_mam_dry_deposition_process_interface.cpp | 41 +++++++------------ .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- externals/mam4xx | 2 +- 4 files changed, 28 insertions(+), 40 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 0dc2caf63baa..3fb8b19abe0c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -10,8 +10,8 @@ namespace scream { namespace { void compute_tendencies( - const int ncol, const int nlev, const mam4::DryDeposition dry_deposition, - const double dt, const MAMDryDep::const_view_1d obklen, + const int ncol, const int nlev, const double dt, + const MAMDryDep::const_view_1d obklen, const MAMDryDep::const_view_1d surfric, const MAMDryDep::const_view_1d landfrac, const MAMDryDep::const_view_1d icefrac, @@ -20,11 +20,11 @@ void compute_tendencies( const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], - const MAMDryDep::view_3d dgncur_awet_, const MAMDryDep::view_3d wet_dens_, + const MAMDryDep::const_view_3d dgncur_awet_, + const MAMDryDep::view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, - const mam_coupling::AerosolState dry_aero, - const mam_coupling::AerosolState wet_aero, MAMDryDep::view_2d aerdepdrycw, - MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_3d tendencies, + const mam_coupling::AerosolState dry_aero, MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_2d aerdepdryis, MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst], MAMDryDep::view_2d rho_, MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] @@ -34,15 +34,14 @@ void compute_tendencies( MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( ncol, nlev); Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { - static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); static constexpr int num_aero_species = mam_coupling::num_aero_species(); - static constexpr int n_land_type = MAMDryDep::n_land_type; const int icol = team.league_rank(); @@ -65,16 +64,18 @@ void compute_tendencies( mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); - mam4::ColumnView dgncur_awet[num_aero_modes], wet_dens[num_aero_modes]; + mam4::ConstColumnView dgncur_awet[num_aero_modes]; + mam4::ColumnView wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { - dgncur_awet[i] = ekat::subview(dgncur_awet_, i, icol); - wet_dens[i] = ekat::subview(wet_dens_, i, icol); + dgncur_awet[i] = ekat::subview(dgncur_awet_, icol, i); + wet_dens[i] = ekat::subview(wet_dens_, icol, i); } mam4::ColumnView rho; rho = ekat::subview(rho_, icol); + static constexpr int n_land_type = MAMDryDep::n_land_type; Real fraction_landuse[n_land_type]; for(int i = 0; i < n_land_type; ++i) { fraction_landuse[i] = fraction_landuse_[i](icol); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index ff29f71e89d8..70260f3ef97f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -94,19 +94,14 @@ void MAMDryDep::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points - const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_mid{{NMODES, COL, LEV}, {num_aero_modes, ncol_, nlev_}}; - - // Layout for tracers. - const int pcnst = mam4::aero_model::pcnst; - FieldLayout scalar4d_q{{COL, LEV, CMP}, {ncol_, nlev_, pcnst}}; - FieldLayout scalar4d_qqcw_tends{{COL, CMP, LEV}, {ncol_, pcnst, nlev_}}; - auto make_layout = [](const std::vector &extents, const std::vector &names) { std::vector tags(extents.size(), CMP); return FieldLayout(tags, extents, names); }; + const int num_aero_modes = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_mid = + make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); using namespace ekat::units; @@ -194,7 +189,7 @@ void MAMDryDep::set_grids( //----------- Variables from other mam4xx processes ------------ // geometric mean wet diameter for number distribution [m] - add_field("dgncur_awet", scalar4d_mid, m, grid_name); + add_field("dgncur_awet", scalar4d_mid, m, grid_name); // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process @@ -244,20 +239,12 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- - - // FIXME: what is the diff between d_qtracers_dt and tendencies???? - // FIXME: we might not need this in FM - add_field("d_qtracers_dt", scalar4d_q, kg / kg / s, grid_name); - // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, 1 / m2 / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, 1 / m2 / s, grid_name); - // FIXME: we might not need this in FM - add_field("Tendencies", scalar4d_qqcw_tends, kg / kg / s, - grid_name); } // set_grids // ================================================================ @@ -309,7 +296,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); - // FIXME: tot or liq? + // FIXME: tot or liq? make notes about it to ensure it is the right one dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); @@ -390,17 +377,16 @@ void MAMDryDep::initialize_impl(const RunType run_type) { .get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - tendencies_ = get_field_out("Tendencies").get_view(); //----------------------------------------------------------------- // Allocate memory //----------------------------------------------------------------- const int pcnst = mam4::aero_model::pcnst; // FIXME: comment what they are and units..... - qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - d_qtracers_dt_ = view_3d("d_qtracers_d_t", ncol_, nlev_, pcnst); + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - rho_ = view_2d("rho", ncol_, nlev_); + rho_ = view_2d("rho", ncol_, nlev_); + d_qtracers_dt_ = view_3d("d_qtracers_dt_", ncol_, nlev_, pcnst); for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { for(int j = 0; j < aerosol_categories_; ++j) { @@ -447,18 +433,19 @@ void MAMDryDep::run_impl(const double dt) { // FIXME: There are some vars that are not declared "REQUIRED" etc. but still // used!!! + // Inputs: // geometric mean wet diameter for number distribution [m] - auto dgncur_awet_ = get_field_out("dgncur_awet").get_view(); + auto dgncur_awet_ = get_field_in("dgncur_awet").get_view(); - compute_tendencies(ncol_, nlev_, dry_deposition, dt, obukhov_length_, + compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, aerodynamical_resistance_, qtracers_, d_qtracers_dt_, fraction_landuse_, // d_qtracers_dt_ is an output - dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, wet_aero_, + dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, // Outputs: - aerdepdrycw_, aerdepdryis_, tendencies_, qqcw_tends_, rho_, - vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); + aerdepdrycw_, aerdepdryis_, qqcw_tends_, rho_, vlc_dry_, + vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); // FIXME: Where is update tends and post processing???? } // run_impl diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 030ffea88789..096311451ace 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -613,7 +613,7 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); - //FIXME : add an assert statement to check id z_surf is zero or not + EKAT_KERNEL_ASSERT_MSG(dry_atm.z_surf == 0, "dry_atm.z_surf must be zero"); PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output team.team_barrier(); // likely necessary to have z_iface up to date diff --git a/externals/mam4xx b/externals/mam4xx index d0ea0c4f68f4..4d16c226c62c 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit d0ea0c4f68f4a939ee73f92bdea797e597eafc3f +Subproject commit 4d16c226c62cc412497fac425359b90a63821efc From 57b4c890e20275440d23bedd411a0747913e6ba8 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 13 Jul 2024 19:05:30 -0400 Subject: [PATCH 156/477] fix second instance of format mismatch --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index c406c1ebd9b4..37cb251d7796 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } From 67200f97ae6744dbdf1bf5f5d19c4b4fcbae6b6a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 16:19:17 -0700 Subject: [PATCH 157/477] Teases out i/o for func --- .../eamxx_mam_dry_deposition_functions.hpp | 46 ++++++++++++------- ...x_mam_dry_deposition_process_interface.cpp | 24 +++++----- ...x_mam_dry_deposition_process_interface.hpp | 4 +- externals/mam4xx | 2 +- 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 3fb8b19abe0c..44742044a0a6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -10,6 +10,7 @@ namespace scream { namespace { void compute_tendencies( + // inputs const int ncol, const int nlev, const double dt, const MAMDryDep::const_view_1d obklen, const MAMDryDep::const_view_1d surfric, @@ -18,14 +19,21 @@ void compute_tendencies( const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, MAMDryDep::view_3d d_qtracers_dt, + MAMDryDep::view_3d qtracers, // FIXME: Make it a constant view MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], const MAMDryDep::const_view_3d dgncur_awet_, - const MAMDryDep::view_3d wet_dens_, + const MAMDryDep::const_view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, - const mam_coupling::AerosolState dry_aero, MAMDryDep::view_2d aerdepdrycw, + const mam_coupling::AerosolState dry_aero, + + // input-outputs + MAMDryDep::view_2d qqcw_[mam4::aero_model::pcnst], + + // outputs + MAMDryDep::view_3d d_qtracers_dt, MAMDryDep::view_2d aerdepdrycw, MAMDryDep::view_2d aerdepdryis, - MAMDryDep::view_2d qqcw_tends_[mam4::aero_model::pcnst], + + // work arrays MAMDryDep::view_2d rho_, MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], @@ -65,7 +73,7 @@ void compute_tendencies( mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); mam4::ConstColumnView dgncur_awet[num_aero_modes]; - mam4::ColumnView wet_dens[num_aero_modes]; + mam4::ConstColumnView wet_dens[num_aero_modes]; for(int i = 0; i < num_aero_modes; ++i) { dgncur_awet[i] = ekat::subview(dgncur_awet_, icol, i); @@ -96,35 +104,39 @@ void compute_tendencies( } } static constexpr int pcnst = mam4::aero_model::pcnst; - mam4::ColumnView qqcw_tends[pcnst]; + mam4::ColumnView qqcw[pcnst]; mam4::ColumnView dqdt_tmp[pcnst]; for(int i = 0; i < pcnst; ++i) { - qqcw_tends[i] = ekat::subview(qqcw_tends_[i], icol); - dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); + qqcw[i] = ekat::subview(qqcw_[i], icol); + dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); } // Extract Prognostics Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { for(int m = 0; m < nmodes; ++m) { - qqcw_tends[mam4::ConvProc::numptrcw_amode(m)][kk] = + qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) - qqcw_tends[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = + qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = progs.q_aero_c[m][a][kk]; } }); // parallel_for nlevs bool ptend_lq[pcnst]; mam4::aero_model_drydep( + // inputs team, fraction_landuse, atm.temperature, atm.pressure, atm.interface_pressure, atm.hydrostatic_dp, - ekat::subview(qtracers, icol), dgncur_awet, wet_dens, qqcw_tends, - obklen[icol], surfric[icol], landfrac[icol], icefrac[icol], - ocnfrac[icol], friction_velocity[icol], - aerodynamical_resistance[icol], ekat::subview(d_qtracers_dt, icol), - ptend_lq, dt, ekat::subview(aerdepdrycw, icol), - ekat::subview(aerdepdryis, icol), rho, vlc_dry, vlc_trb, vlc_grv, - dqdt_tmp); + ekat::subview(qtracers, icol), dgncur_awet, wet_dens, obklen[icol], + surfric[icol], landfrac[icol], icefrac[icol], ocnfrac[icol], + friction_velocity[icol], aerodynamical_resistance[icol], dt, + // input-outputs + qqcw, + // outputs + ekat::subview(d_qtracers_dt, icol), ptend_lq, + ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), + // work arrays + rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); }); // parallel_for for ncols } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 70260f3ef97f..e8218fbe5c8a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -188,15 +188,16 @@ void MAMDryDep::set_grids( add_field("ocean_fraction", scalar2d, nondim, grid_name); //----------- Variables from other mam4xx processes ------------ - // geometric mean wet diameter for number distribution [m] + // Geometric mean wet diameter for number distribution [m] add_field("dgncur_awet", scalar4d_mid, m, grid_name); + // Wet density of interstitial aerosol [kg/m3] + add_field("wetdens", scalar4d_mid, kg / m3, grid_name); + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- - add_field("wetdens", scalar4d_mid, kg / m3, grid_name); - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for(int m = 0; m < num_aero_modes; ++m) { @@ -239,6 +240,7 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- + // FIXME: These are diagnostics, remove them from FM after initial evaluation // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, 1 / m2 / s, grid_name); @@ -372,7 +374,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: We might need to get rid of few of these fields // as we do not need them in FM - wet_dens_ = get_field_out("wetdens").get_view(); aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") .get_view(); aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") @@ -397,7 +398,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { } for(int i = 0; i < pcnst; ++i) { - Kokkos::resize(qqcw_tends_[i], ncol_, nlev_); + Kokkos::resize(qqcw_[i], ncol_, nlev_); Kokkos::resize(dqdt_tmp_[i], ncol_, nlev_); } @@ -424,27 +425,24 @@ void MAMDryDep::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - const DryDep::Config process_config; - - DryDep dry_deposition; - const mam4::AeroConfig aero_config; - dry_deposition.init(aero_config, process_config); - // FIXME: There are some vars that are not declared "REQUIRED" etc. but still // used!!! // Inputs: // geometric mean wet diameter for number distribution [m] auto dgncur_awet_ = get_field_in("dgncur_awet").get_view(); + auto wet_dens_ = get_field_in("wetdens").get_view(); compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, - aerodynamical_resistance_, qtracers_, d_qtracers_dt_, + aerodynamical_resistance_, qtracers_, fraction_landuse_, // d_qtracers_dt_ is an output dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, + // Inouts-outputs + qqcw_, // Outputs: - aerdepdrycw_, aerdepdryis_, qqcw_tends_, rho_, vlc_dry_, + d_qtracers_dt_, aerdepdrycw_, aerdepdryis_, rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); // FIXME: Where is update tends and post processing???? diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index fdfb387524d8..d7ca643b9bbe 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -50,7 +50,6 @@ class MAMDryDep final : public scream::AtmosphereProcess { // inputs // FIXME: collect all inputs and outputs together view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; - view_3d wet_dens_; view_2d rho_; view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; @@ -58,10 +57,9 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d dqdt_tmp_[mam4::aero_model::pcnst]; - view_3d tendencies_; view_2d aerdepdrycw_; view_2d aerdepdryis_; - view_2d qqcw_tends_[mam4::aero_model::pcnst]; + view_2d qqcw_[mam4::aero_model::pcnst]; const_view_1d obukhov_length_; const_view_1d surface_friction_velocty_; diff --git a/externals/mam4xx b/externals/mam4xx index 4d16c226c62c..036b4c83b289 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 4d16c226c62cc412497fac425359b90a63821efc +Subproject commit 036b4c83b289ff6c4258018a92c87e4defa628a9 From 90eaa5848947fc620dbf1141b5694c2472b0b8db Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Jul 2024 16:35:44 -0700 Subject: [PATCH 158/477] A lot of cleanup-Removes vars from hpp, cleans i/o for functions --- ...x_mam_dry_deposition_process_interface.cpp | 72 +++++++++++-------- ...x_mam_dry_deposition_process_interface.hpp | 18 ++--- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index e8218fbe5c8a..9b99e6fe78a4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -303,17 +303,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); - obukhov_length_ = get_field_in("Obukhov_length").get_view(); - land_fraction_ = get_field_in("land_fraction").get_view(); - ice_fraction_ = get_field_in("ice_fraction").get_view(); - ocean_fraction_ = get_field_in("ocean_fraction").get_view(); - friction_velocity_ = - get_field_in("friction_velocity").get_view(); - aerodynamical_resistance_ = - get_field_in("aerodynamical_resistance").get_view(); - surface_friction_velocty_ = - get_field_in("surface_friction_velocty").get_view(); - // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; dry_atm_.z_iface = buffer_.z_iface; @@ -367,18 +356,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } - // ------------------------------------------------------------- - // Output fields for the process - // ------------------------------------------------------------- - - // FIXME: We might need to get rid of few of these fields - // as we do not need them in FM - - aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") - .get_view(); - aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") - .get_view(); - //----------------------------------------------------------------- // Allocate memory //----------------------------------------------------------------- @@ -428,22 +405,55 @@ void MAMDryDep::run_impl(const double dt) { // FIXME: There are some vars that are not declared "REQUIRED" etc. but still // used!!! - // Inputs: - // geometric mean wet diameter for number distribution [m] + // ------------------------------------------------------------- + // Inputs fields for the process + // ------------------------------------------------------------- + + // Geometric mean wet diameter for number distribution [m] auto dgncur_awet_ = get_field_in("dgncur_awet").get_view(); - auto wet_dens_ = get_field_in("wetdens").get_view(); + // Wet density of interstitial aerosol [kg/m3] + auto wet_dens_ = get_field_in("wetdens").get_view(); + // Obukhov length [m] + auto obukhov_length_ = + get_field_in("Obukhov_length").get_view(); + // Land fraction [unitless] + auto land_fraction_ = get_field_in("land_fraction").get_view(); + // Ice fraction [unitless] + auto ice_fraction_ = get_field_in("ice_fraction").get_view(); + // Ocean fraction [unitless] + auto ocean_fraction_ = + get_field_in("ocean_fraction").get_view(); + // Friction velocity from land model [m/s] + auto friction_velocity_ = + get_field_in("friction_velocity").get_view(); + // Aerodynamical resistance from land model [s/m] + auto aerodynamical_resistance_ = + get_field_in("aerodynamical_resistance").get_view(); + // Sfc friction velocity [m/s] + auto surface_friction_velocty_ = + get_field_in("surface_friction_velocty").get_view(); + + // ------------------------------------------------------------- + // Output fields for the process + // ------------------------------------------------------------- + // Surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] + auto aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + .get_view(); + // Surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] + auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + .get_view(); compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, - aerodynamical_resistance_, qtracers_, - fraction_landuse_, // d_qtracers_dt_ is an output + aerodynamical_resistance_, qtracers_, fraction_landuse_, dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, // Inouts-outputs qqcw_, - // Outputs: - d_qtracers_dt_, aerdepdrycw_, aerdepdryis_, rho_, vlc_dry_, - vlc_trb_, vlc_grv_, dqdt_tmp_); + // Outputs + d_qtracers_dt_, aerdepdrycw_, aerdepdryis_, + // work arrays + rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); // FIXME: Where is update tends and post processing???? } // run_impl diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index d7ca643b9bbe..58267cb919a7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -20,12 +20,12 @@ class MAMDryDep final : public scream::AtmosphereProcess { using view_2d = Field::view_dev_t; using view_3d = Field::view_dev_t; using const_view_1d = Field::view_dev_t; - using const_view_2d = Field::view_dev_t; using const_view_3d = Field::view_dev_t; - static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); - static constexpr int aerosol_categories_ = 4; // FIXME: get it from mam4xx - static constexpr int n_land_type = mam4::DryDeposition::n_land_type; + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int aerosol_categories_ = + mam4::DryDeposition::aerosol_categories; + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; private: // number of horizontal columns and vertical levels @@ -57,18 +57,8 @@ class MAMDryDep final : public scream::AtmosphereProcess { view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d dqdt_tmp_[mam4::aero_model::pcnst]; - view_2d aerdepdrycw_; - view_2d aerdepdryis_; view_2d qqcw_[mam4::aero_model::pcnst]; - const_view_1d obukhov_length_; - const_view_1d surface_friction_velocty_; - const_view_1d land_fraction_; - const_view_1d ice_fraction_; - const_view_1d ocean_fraction_; - const_view_1d friction_velocity_; - const_view_1d aerodynamical_resistance_; - public: using KT = ekat::KokkosTypes; From 811f80b6671d65f147a1e8db644ecb8efa5b8e9c Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 12 Jul 2024 16:42:47 -0500 Subject: [PATCH 159/477] rename pyeamxx to pyscream --- components/eamxx/src/python/CMakeLists.txt | 4 ++-- components/eamxx/src/python/pyatmproc.hpp | 2 +- components/eamxx/src/python/pyeamxx.cpp | 2 +- components/eamxx/src/python/pyeamxx.hpp | 6 +++--- components/eamxx/src/python/pyeamxx/__init__.py | 16 ++++++++-------- components/eamxx/src/python/pygrid.hpp | 4 ++-- components/eamxx/src/python/pyproject.toml | 6 +++--- components/eamxx/src/python/readme | 6 +++--- .../eamxx/tests/python/pyp3/CMakeLists.txt | 2 +- .../eamxx/tests/python/pyp3/p3_standalone_py | 12 ++++++------ 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 43e2db73e533..41eae6d65463 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,8 +1,8 @@ find_package(pybind11 REQUIRED) find_package(mpi4py REQUIRED) -pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC +pybind11_add_module(pyscream pyscream.cpp) +target_link_libraries(pyscream PUBLIC mpi4py scream_share scream_io diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index 98d7b9a525ed..9e45bc513f1b 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -8,7 +8,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" -#include "pyeamxx.hpp" +#include "pyscream.hpp" #include diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index c4ec360d80a1..2efb8b6f8df9 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -45,7 +45,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyeamxx,m) { +PYBIND11_MODULE (pyscream,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/pyeamxx.hpp index 59371844a763..781c195919dd 100644 --- a/components/eamxx/src/python/pyeamxx.hpp +++ b/components/eamxx/src/python/pyeamxx.hpp @@ -1,5 +1,5 @@ -#ifndef PYEAMXX_HPP -#define PYEAMXX_HPP +#ifndef PYSCREAM_HPP +#define PYSCREAM_HPP #include "physics/register_physics.hpp" #include "diagnostics/register_diagnostics.hpp" @@ -27,4 +27,4 @@ struct PySession { } // namespace scream -#endif // PYEAMXX_HPP +#endif // PYSCREAM_HPP diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py index 75a36e148291..98c31744eb9f 100644 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ b/components/eamxx/src/python/pyeamxx/__init__.py @@ -1,15 +1,15 @@ """ This file will serve as a way to organize and expose - libpyeamxx internals to the rest of pyeamxx + libpyscream internals to the rest of pyscream """ -from libpyeamxx.libpyeamxx_ext import AtmProc -from libpyeamxx.libpyeamxx_ext import Grid -from libpyeamxx.libpyeamxx_ext import ParameterList -from libpyeamxx.libpyeamxx_ext import init -from libpyeamxx.libpyeamxx_ext import Field -from libpyeamxx.libpyeamxx_ext import P3 -from libpyeamxx.libpyeamxx_ext import finalize +from libpyscream.libpyscream_ext import AtmProc +from libpyscream.libpyscream_ext import Grid +from libpyscream.libpyscream_ext import ParameterList +from libpyscream.libpyscream_ext import init +from libpyscream.libpyscream_ext import Field +from libpyscream.libpyscream_ext import P3 +from libpyscream.libpyscream_ext import finalize __all__ = [ diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp index 102cf6037c62..e49a0443a345 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/pygrid.hpp @@ -3,7 +3,7 @@ #include "share/grid/mesh_free_grids_manager.hpp" -#include "pyeamxx.hpp" +#include "pyscream.hpp" #include @@ -14,7 +14,7 @@ namespace scream { inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) { EKAT_REQUIRE_MSG (PySession::get().inited, - "Error! You did not initialize pyeamxx, or you already finalized it!\n"); + "Error! You did not initialize pyscream, or you already finalized it!\n"); auto& comm = PySession::get().comm; ekat::ParameterList gm_params; std::vector grids_names = {"Physics"}; diff --git a/components/eamxx/src/python/pyproject.toml b/components/eamxx/src/python/pyproject.toml index fa5d6fc44817..490ea99c143e 100644 --- a/components/eamxx/src/python/pyproject.toml +++ b/components/eamxx/src/python/pyproject.toml @@ -3,15 +3,15 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "pyeamxx" +name = "pyscream" version = "0.0.2" dependencies = ["numpy", "mpi4py"] [tool.setuptools.packages.find] where = ["."] -include = ["pyeamxx", "libpyeamxx"] +include = ["pyscream", "libpyscream"] exclude = ["build_src", "tests"] namespaces = true [tool.setuptools.package-data] -"libpyeamxx" = ["*.so*"] +"libpyscream" = ["*.so*"] diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme index f5ed48f058ee..c16c32a5d217 100644 --- a/components/eamxx/src/python/readme +++ b/components/eamxx/src/python/readme @@ -1,7 +1,7 @@ INFO: - EAMxx python bindings -- pyeamxx is where we will house the python code -- libpyeamxx is where we will house the extensions +- pyscream is where we will house the python code +- libpyscream is where we will house the extensions - packaging moved to https://github.com/mahf708/experimental-scream-feedstock TODO: @@ -11,5 +11,5 @@ TODO: - decide archs/pythons/mpis to tgt USER: -- conda install pyeamxx -c mahf708/label/$mac (mac is chrysalis or pm-cpu) +- conda install pyscream -c mahf708/label/$mac (mac is chrysalis or pm-cpu) - see example in components/eamxx/tests/python/pyp3 diff --git a/components/eamxx/tests/python/pyp3/CMakeLists.txt b/components/eamxx/tests/python/pyp3/CMakeLists.txt index 7142be8dd8f8..945f15faaf65 100644 --- a/components/eamxx/tests/python/pyp3/CMakeLists.txt +++ b/components/eamxx/tests/python/pyp3/CMakeLists.txt @@ -53,6 +53,6 @@ foreach (rank IN LISTS MpiRanks) SRC_FILE p3_standalone_cxx.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc TGT_FILE p3_standalone_py.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_cxx${suffix} ${FIXTURES_BASE_NAME}_py${suffix} - LABELS pyeamxx + LABELS pyscream ) endforeach() diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index 0db6bcfcef61..34747943d081 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -5,7 +5,7 @@ import sys # Add path to scream libs sys.path.append('@SCREAM_BASE_DIR@/scripts') -# Add path to pyeamxx libs +# Add path to pyscream libs sys.path.append('@CMAKE_BINARY_DIR@/src/python') # Without these, and manual init/finalize, on my laptop I get @@ -15,7 +15,7 @@ mpi4py.rc.initialize = False # do not initialize MPI automatically mpi4py.rc.finalize = False # do not finalize MPI automatically from mpi4py import MPI -import pyeamxx +import pyscream from pathlib import Path from utils import ensure_yaml @@ -38,9 +38,9 @@ def main (): # Create the grid ncols = 218 nlevs = 72 - pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) + pyscream.create_grids_manager(ncols,nlevs,str(ic_file)) - p3 = pyeamxx.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') + p3 = pyscream.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') params = p3.get_params() old = params.get_dbl('max_total_ni') print (f"max_total_ni: {params.get_dbl('max_total_ni')}") @@ -64,7 +64,7 @@ if __name__ == "__main__": # This level of indirection ensures all pybind structs are destroyed # before we finalize eamxx (and hence kokkos) MPI.Init() - pyeamxx.init() + pyscream.init() main () - pyeamxx.finalize() + pyscream.finalize() MPI.Finalize() From c9af8dc8f11f853740303ed2cfddac25cd66d2d0 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 12 Jul 2024 16:54:10 -0500 Subject: [PATCH 160/477] more renaming --- components/eamxx/src/python/CMakeLists.txt | 13 +---------- .../src/python/libpyscream/CMakeLists.txt | 12 ++++++++++ .../python/{ => libpyscream}/pyatmproc.hpp | 2 +- .../src/python/{ => libpyscream}/pyfield.hpp | 0 .../src/python/{ => libpyscream}/pygrid.hpp | 2 +- .../python/{ => libpyscream}/pyparamlist.hpp | 0 .../pyscream_ext.cpp} | 2 +- .../pyscream_ext.hpp} | 0 .../src/python/{ => libpyscream}/pyutils.hpp | 0 .../eamxx/src/python/pyeamxx/__init__.py | 23 ------------------- .../eamxx/src/python/pyscream/__init__.py | 10 ++++++++ 11 files changed, 26 insertions(+), 38 deletions(-) create mode 100644 components/eamxx/src/python/libpyscream/CMakeLists.txt rename components/eamxx/src/python/{ => libpyscream}/pyatmproc.hpp (99%) rename components/eamxx/src/python/{ => libpyscream}/pyfield.hpp (100%) rename components/eamxx/src/python/{ => libpyscream}/pygrid.hpp (97%) rename components/eamxx/src/python/{ => libpyscream}/pyparamlist.hpp (100%) rename components/eamxx/src/python/{pyeamxx.cpp => libpyscream/pyscream_ext.cpp} (97%) rename components/eamxx/src/python/{pyeamxx.hpp => libpyscream/pyscream_ext.hpp} (100%) rename components/eamxx/src/python/{ => libpyscream}/pyutils.hpp (100%) delete mode 100644 components/eamxx/src/python/pyeamxx/__init__.py create mode 100644 components/eamxx/src/python/pyscream/__init__.py diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 41eae6d65463..4c8adf40fca7 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,12 +1 @@ -find_package(pybind11 REQUIRED) -find_package(mpi4py REQUIRED) - -pybind11_add_module(pyscream pyscream.cpp) -target_link_libraries(pyscream PUBLIC - mpi4py - scream_share - scream_io - diagnostics - eamxx_physics - scream_test_support -) +add_subdirectory(libpyscream) \ No newline at end of file diff --git a/components/eamxx/src/python/libpyscream/CMakeLists.txt b/components/eamxx/src/python/libpyscream/CMakeLists.txt new file mode 100644 index 000000000000..103c6a80b888 --- /dev/null +++ b/components/eamxx/src/python/libpyscream/CMakeLists.txt @@ -0,0 +1,12 @@ +find_package(pybind11 REQUIRED) +find_package(mpi4py REQUIRED) + +pybind11_add_module(pyscream_ext pyscream_ext.cpp) +target_link_libraries(pyscream_ext PUBLIC + mpi4py + scream_share + scream_io + diagnostics + eamxx_physics + scream_test_support +) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp similarity index 99% rename from components/eamxx/src/python/pyatmproc.hpp rename to components/eamxx/src/python/libpyscream/pyatmproc.hpp index 9e45bc513f1b..0acc371c1f71 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -8,7 +8,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" -#include "pyscream.hpp" +#include "pyscream_ext.hpp" #include diff --git a/components/eamxx/src/python/pyfield.hpp b/components/eamxx/src/python/libpyscream/pyfield.hpp similarity index 100% rename from components/eamxx/src/python/pyfield.hpp rename to components/eamxx/src/python/libpyscream/pyfield.hpp diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp similarity index 97% rename from components/eamxx/src/python/pygrid.hpp rename to components/eamxx/src/python/libpyscream/pygrid.hpp index e49a0443a345..e8adfa3bbc49 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -3,7 +3,7 @@ #include "share/grid/mesh_free_grids_manager.hpp" -#include "pyscream.hpp" +#include "pyscream_ext.hpp" #include diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/libpyscream/pyparamlist.hpp similarity index 100% rename from components/eamxx/src/python/pyparamlist.hpp rename to components/eamxx/src/python/libpyscream/pyparamlist.hpp diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp similarity index 97% rename from components/eamxx/src/python/pyeamxx.cpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.cpp index 2efb8b6f8df9..e0ad9ceec260 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp @@ -45,7 +45,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyscream,m) { +PYBIND11_MODULE (pyscream_ext,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/libpyscream/pyscream_ext.hpp similarity index 100% rename from components/eamxx/src/python/pyeamxx.hpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.hpp diff --git a/components/eamxx/src/python/pyutils.hpp b/components/eamxx/src/python/libpyscream/pyutils.hpp similarity index 100% rename from components/eamxx/src/python/pyutils.hpp rename to components/eamxx/src/python/libpyscream/pyutils.hpp diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py deleted file mode 100644 index 98c31744eb9f..000000000000 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" - This file will serve as a way to organize and expose - libpyscream internals to the rest of pyscream -""" - -from libpyscream.libpyscream_ext import AtmProc -from libpyscream.libpyscream_ext import Grid -from libpyscream.libpyscream_ext import ParameterList -from libpyscream.libpyscream_ext import init -from libpyscream.libpyscream_ext import Field -from libpyscream.libpyscream_ext import P3 -from libpyscream.libpyscream_ext import finalize - - -__all__ = [ - 'init', - 'finalize', - 'AtmProc', - 'Grid', - 'ParameterList', - 'Field', - 'P3', -] diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py new file mode 100644 index 000000000000..8159f5da47c7 --- /dev/null +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -0,0 +1,10 @@ +""" + This file will serve as a way to organize and expose + libpyscream internals to the rest of pyscream +""" + +from libpyscream import pyscream_ext as pyscream + +__all__ = [ + "pyscream" +] From 98ee753002733fa2e377ad5b38b8d92b4d666eaa Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 12:06:01 -0500 Subject: [PATCH 161/477] skip cosp when building python --- components/eamxx/src/physics/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e0e89e60f80d..e5bce027bcc4 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -7,7 +7,11 @@ add_subdirectory(share) add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) - add_subdirectory(cosp) + if (EAMXX_ENABLE_PYBIND) + message(STATUS "WARNING: skipping COSP when buidling Python") + else() + add_subdirectory(cosp) + endif() else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() From 86d53719be0079009b6b02637915ecf321fb93af Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 14:02:39 -0500 Subject: [PATCH 162/477] Add geo_data_source --- components/eamxx/src/python/libpyscream/pygrid.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index e8adfa3bbc49..93a8a5395dba 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -25,6 +25,7 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo gm_params.set("grids_names",grids_names); if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } From 628e401b45f4e395c28dec843742ae17da1b23a7 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 23:04:05 -0500 Subject: [PATCH 163/477] dump fields and reorg things --- components/eamxx/src/physics/CMakeLists.txt | 6 +----- components/eamxx/src/python/CMakeLists.txt | 2 +- .../eamxx/src/python/libpyscream/pyatmproc.hpp | 9 +++++++++ components/eamxx/src/python/libpyscream/pygrid.hpp | 1 - components/eamxx/src/python/pyscream/__init__.py | 14 ++++++++++++-- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e5bce027bcc4..e0e89e60f80d 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -7,11 +7,7 @@ add_subdirectory(share) add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) - if (EAMXX_ENABLE_PYBIND) - message(STATUS "WARNING: skipping COSP when buidling Python") - else() - add_subdirectory(cosp) - endif() + add_subdirectory(cosp) else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 4c8adf40fca7..d19bc05c240e 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1 +1 @@ -add_subdirectory(libpyscream) \ No newline at end of file +add_subdirectory(libpyscream) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 0acc371c1f71..5248ccf1f965 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -131,6 +131,14 @@ struct PyAtmProc { return pybind11::cast(missing); } + pybind11::list dump_fields() { + std::vector all_fields; + for (auto it : fields) { + all_fields.push_back(it.first); + } + return pybind11::cast(all_fields); + } + void setup_output (const std::string& yaml_file) { auto comm = PySession::get().comm; @@ -176,6 +184,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) .def("read_ic",&PyAtmProc::read_ic); + .def("dump_fields",&PyAtmProc::dump_fields); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index 93a8a5395dba..e8adfa3bbc49 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -25,7 +25,6 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo gm_params.set("grids_names",grids_names); if (latlon_nc_file!="") { - gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py index 8159f5da47c7..e76f52170fb3 100644 --- a/components/eamxx/src/python/pyscream/__init__.py +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -3,8 +3,18 @@ libpyscream internals to the rest of pyscream """ -from libpyscream import pyscream_ext as pyscream +from libpyscream.pyscream_ext import init +from libpyscream.pyscream_ext import finalize +from libpyscream.pyscream_ext import Field +from libpyscream.pyscream_ext import AtmProc +from libpyscream.pyscream_ext import ParameterList +from libpyscream.pyscream_ext import create_grids_manager __all__ = [ - "pyscream" + "init", + "finalize", + "Field", + "AtmProc", + "ParameterList", + "create_grids_manager", ] From 2599ce5eedbabcd12ef6ec78d3b2ba14c2097bef Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 14 Jul 2024 09:56:44 -0500 Subject: [PATCH 164/477] fix minor typo in def dumpfields --- components/eamxx/src/python/libpyscream/pyatmproc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 5248ccf1f965..000a7d64c9a9 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -183,7 +183,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("get_params",&PyAtmProc::get_params) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) - .def("read_ic",&PyAtmProc::read_ic); + .def("read_ic",&PyAtmProc::read_ic) .def("dump_fields",&PyAtmProc::dump_fields); } } // namespace scream From 3e0bb221b1c5308b5567a268e2cfe5b565a031fd Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 14 Jul 2024 11:48:48 -0500 Subject: [PATCH 165/477] add geo data source --- components/eamxx/src/python/libpyscream/pygrid.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index e8adfa3bbc49..1c0d56eb08b3 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -23,8 +23,10 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo pl.set("number_of_global_columns",ncols); pl.set("number_of_vertical_levels",nlevs); gm_params.set("grids_names",grids_names); + gm_params.set("geo_data_source",std::string("CREATE_EMPTY_DATA")); if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } From f12a60023f23ab9b5387798b991d9b33f540ad9d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 14 Jul 2024 15:53:13 -0700 Subject: [PATCH 166/477] Reduces vlc_trb to real from view_1d and other cleanup --- .../eamxx_mam_dry_deposition_functions.hpp | 11 ++-- ...x_mam_dry_deposition_process_interface.cpp | 52 +++++++++---------- ...x_mam_dry_deposition_process_interface.hpp | 9 ++-- .../single-process/mam/aci/CMakeLists.txt | 1 + .../single-process/mam/drydep/CMakeLists.txt | 3 +- externals/mam4xx | 2 +- 6 files changed, 42 insertions(+), 36 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 44742044a0a6..ad3c023e3878 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -37,12 +37,14 @@ void compute_tendencies( MAMDryDep::view_2d rho_, MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], - MAMDryDep::view_2d vlc_trb_[mam4::AeroConfig::num_modes()] + MAMDryDep::view_1d vlc_trb_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + // FIXME: WHY we are using a new policy here?? can't we get it from the + // run_impl?? const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( ncol, nlev); @@ -93,13 +95,14 @@ void compute_tendencies( // Kokkos::View. Solution: Use ColumnView in drydep.hpp as well. static constexpr int nmodes = mam4::AeroConfig::num_modes(); mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_], - vlc_trb[nmodes][MAMDryDep::aerosol_categories_], vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; + Real vlc_trb[nmodes][MAMDryDep::aerosol_categories_]; + for(int i = 0; i < nmodes; ++i) { for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { vlc_dry[i][j] = ekat::subview(vlc_dry_[i][j], icol); - vlc_trb[i][j] = ekat::subview(vlc_trb_[i][j], icol); + vlc_trb[i][j] = vlc_trb_[i][j](icol); vlc_grv[i][j] = ekat::subview(vlc_grv_[i][j], icol); } } @@ -142,4 +145,4 @@ void compute_tendencies( } // namespace } // namespace scream -#endif \ No newline at end of file +#endif diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 9b99e6fe78a4..4160b467f340 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -1,6 +1,6 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" -// ACI functions are stored in the following hpp file +// Drydep functions are stored in the following hpp file #include #include "mam4xx/drydep.hpp" //FIXME: Do we need it here??? @@ -84,7 +84,6 @@ void MAMDryDep::set_grids( using namespace ShortFieldTagsNames; // Layout for 2D (2d horiz) variable - // FIXME: Fix layouts based on new format const FieldLayout scalar2d{{COL}, {ncol_}}; // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and @@ -113,29 +112,30 @@ void MAMDryDep::set_grids( auto m3 = m * m * m; // meter cubed // -------------------------------------------------------------------------- - // These variables are "required" or pure inputs for the process + // These variables are "Required" or pure inputs for the process // -------------------------------------------------------------------------- // ----------- Atmospheric quantities ------------- - // Specific humidity [kg/kg] + // Specific humidity [kg/kg](Require only for building DS) add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); - // Cloud liquid mass mixing ratio [kg/kg] + // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); - // Cloud ice mass mixing ratio [kg/kg] + // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); - // Cloud liquid number mixing ratio [1/kg] + // Cloud liquid number mixing ratio [1/kg](Require only for building DS) add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); - // Cloud ice number mixing ratio [1/kg] + // Cloud ice number mixing ratio [1/kg](Require only for building DS) add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); - // Vertical pressure velocity [Pa/s] at midpoints + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) add_field("omega", scalar3d_mid, Pa / s, grid_name); // Total pressure [Pa] at midpoints @@ -147,26 +147,25 @@ void MAMDryDep::set_grids( // Layer thickness(pdel) [Pa] at midpoints add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - // Planetary boundary layer height [m] + // Planetary boundary layer height [m] (Require only for building DS) add_field("pbl_height", scalar2d, m, grid_name); static constexpr auto m2 = m * m; static constexpr auto s2 = s * s; - // Surface geopotential [m2/s2] + // Surface geopotential [m2/s2] (Require only for building DS) add_field("phis", scalar2d, m2 / s2, grid_name); //----------- Variables from microphysics scheme ------------- - // Total cloud fraction [fraction] - // FIXME: Is is cldfrac_liq instead? find out + // Total cloud fraction [fraction] (Require only for building DS) add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); //----------- Variables from coupler (land component)--------- // Obukhov length [m] add_field("Obukhov_length", scalar2d, m, grid_name); - // Surface friction velocty [m] + // Surface friction velocty or ustar[m/s] add_field("surface_friction_velocty", scalar2d, m / s, grid_name); // Land fraction [fraction] @@ -294,11 +293,10 @@ void MAMDryDep::initialize_impl(const RunType run_type) { wet_atm_.ni = get_field_in("ni").get_view(); // Populate the dry atmosphere state with views from fields - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); - // FIXME: tot or liq? make notes about it to ensure it is the right one + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); @@ -370,7 +368,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { for(int j = 0; j < aerosol_categories_; ++j) { Kokkos::resize(vlc_dry_[i][j], ncol_, nlev_); Kokkos::resize(vlc_grv_[i][j], ncol_, nlev_); - Kokkos::resize(vlc_trb_[i][j], ncol_, nlev_); + Kokkos::resize(vlc_trb_[i][j], ncol_); } } @@ -381,6 +379,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { static constexpr int n_land_type = mam4::DryDeposition::n_land_type; for(int i = 0; i < n_land_type; ++i) { + // FIXME: This should come from a file reading Kokkos::resize(fraction_landuse_[i], ncol_); } @@ -389,7 +388,8 @@ void MAMDryDep::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - // FIXME: Where is post processing functor???? + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // initialize_impl // ========================================================================================= @@ -402,9 +402,6 @@ void MAMDryDep::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // FIXME: There are some vars that are not declared "REQUIRED" etc. but still - // used!!! - // ------------------------------------------------------------- // Inputs fields for the process // ------------------------------------------------------------- @@ -429,7 +426,7 @@ void MAMDryDep::run_impl(const double dt) { // Aerodynamical resistance from land model [s/m] auto aerodynamical_resistance_ = get_field_in("aerodynamical_resistance").get_view(); - // Sfc friction velocity [m/s] + // Sfc friction velocity or ustar [m/s] auto surface_friction_velocty_ = get_field_in("surface_friction_velocty").get_view(); @@ -455,6 +452,9 @@ void MAMDryDep::run_impl(const double dt) { // work arrays rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); - // FIXME: Where is update tends and post processing???? + // call post processing to convert dry mixing ratios to wet mixing ratios + // and update the state + // Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + // Kokkos::fence(); // wait before returning to calling function } // run_impl } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 58267cb919a7..867a81d9fb40 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -50,10 +50,10 @@ class MAMDryDep final : public scream::AtmosphereProcess { // inputs // FIXME: collect all inputs and outputs together view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; + view_1d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d rho_; view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; - view_2d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; view_2d dqdt_tmp_[mam4::aero_model::pcnst]; @@ -126,7 +126,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { // for atmosphere compute_vertical_layer_heights(team, dry_atm_pre_, i); compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - } // operator() + } // Preprocess operator() // local variables for preprocess struct // number of horizontal columns and vertical levels @@ -136,7 +136,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // Preprocess // Atmosphere processes often have a post-processing step prepares output // from this process for the Field Manager. This functor implements this @@ -165,7 +165,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { const int i = team.league_rank(); // column index compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, wet_aero_post_, i); - } // operator() + } // operator() Postprocess // number of horizontal columns and vertical levels int ncol_post_, nlev_post_; @@ -179,6 +179,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { private: // pre- and postprocessing scratch pads Preprocess preprocess_; + Postprocess postprocess_; }; // MAMDryDep } // namespace scream diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 0d5d60ee7d5a..490dc389d952 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -24,6 +24,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) diff --git a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt index 50d030cdc2cf..7bc5010370a4 100644 --- a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt @@ -21,4 +21,5 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) # Ensure test input files are present in the data dir -GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) diff --git a/externals/mam4xx b/externals/mam4xx index 036b4c83b289..f7f445dcb646 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 036b4c83b289ff6c4258018a92c87e4defa628a9 +Subproject commit f7f445dcb6464105a1719e16b68928e583cc8ec6 From ef6e76653875f7e7c49c1f6649b8e41f39a3682d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:48:25 +0000 Subject: [PATCH 167/477] Bump actions/setup-python from 5.1.0 to 5.1.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ccca0c479f26..9ea25ae1864e 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index abd8b92e4f6c..e6fb53ba29f5 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 5ecdf6dec00c..e0a8e19f9c01 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" - name: Run unit tests From 8fc5d8f338eb3d3c9d501ec2f17e644c31e4bb4f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 15 Jul 2024 12:02:37 -0700 Subject: [PATCH 168/477] All the printouts for thevalidation --- .../eamxx_mam_dry_deposition_functions.hpp | 23 ++++++++++++++++++- ...x_mam_dry_deposition_process_interface.cpp | 6 +++-- .../eamxx/src/physics/mam/mam_coupling.hpp | 20 ++++++++-------- .../src/share/io/scream_scorpio_interface.cpp | 4 ++-- .../single-process/mam/drydep/CMakeLists.txt | 3 ++- .../single-process/mam/drydep/input.yaml | 14 ++--------- externals/mam4xx | 2 +- 7 files changed, 43 insertions(+), 29 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index ad3c023e3878..ff876cdcf425 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -42,12 +42,15 @@ void compute_tendencies( MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { + auto printb = [](const std::string &name, const double &val) { + std::cout << name << ":" << std::setprecision(15) << val << std::endl; + }; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); // FIXME: WHY we are using a new policy here?? can't we get it from the // run_impl?? const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( - ncol, nlev); + 1, nlev); Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { static constexpr int num_aero_species = @@ -126,6 +129,7 @@ void compute_tendencies( } }); // parallel_for nlevs bool ptend_lq[pcnst]; + printb("bef:qqcw:", qqcw[23](63)); mam4::aero_model_drydep( // inputs team, fraction_landuse, atm.temperature, atm.pressure, @@ -140,8 +144,25 @@ void compute_tendencies( ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), // work arrays rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); + printb("aft:qqcw:", qqcw[23](63)); }); // parallel_for for ncols } + +void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { + Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, + 0.16803558403621365E-001, 0.18076055155371872E-001, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.91803784897907303E+000, 0.17186036997038400E-002, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.18448503115578840E-001}; + + for(int icol = 0; icol < ncol; ++icol) { + for(int kk = 0; kk < 11; ++kk) { + flu[kk](icol) = temp[kk]; + } + } +} + } // namespace } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 4160b467f340..a38ad080a3d4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -100,7 +100,7 @@ void MAMDryDep::set_grids( }; const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); + make_layout({ncol_, num_aero_modes, nlev_}, {"ncol", "num_modes", "lev"}); using namespace ekat::units; @@ -396,7 +396,7 @@ void MAMDryDep::initialize_impl(const RunType run_type) { void MAMDryDep::run_impl(const double dt) { using DryDep = mam4::DryDeposition; const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -440,6 +440,8 @@ void MAMDryDep::run_impl(const double dt) { auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); + populated_fraction_landuse(fraction_landuse_, ncol_); + // std::cout<<"fraction_landuse_:"< Date: Mon, 15 Jul 2024 16:02:16 -0600 Subject: [PATCH 169/477] Ensure that Kokkos::deep_copy is called outside of parallel_for. --- .../mam/eamxx_mam_wetscav_process_interface.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 91e69443b588..bbd99a66cfb0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -461,6 +461,14 @@ void MAMWetscav::run_impl(const double dt) { // inside a parallel_for. const int nlev = nlev_; + // Zero out tendencies otherwise, they are initialized to junk values + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(tends.n_mode_i[m], 0); + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + Kokkos::deep_copy(tends.q_aero_i[m][a], 0); + } + } + // Loop over atmosphere columns Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { @@ -506,14 +514,6 @@ void MAMWetscav::run_impl(const double dt) { auto wetdens_icol = ekat::subview(wetdens, icol); const auto prain_icol = ekat::subview(prain, icol); - // Zero out tendencies otherwise, they are initialized to junk values - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - Kokkos::deep_copy(tends.n_mode_i[m], 0); - for(int a = 0; a < mam4::num_species_mode(m); ++a) { - Kokkos::deep_copy(tends.q_aero_i[m][a], 0); - } - } - mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, // inputs From 9cb990cfd6976cb989a34dc49a8a1bb33f61637e Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 15 Jul 2024 16:16:33 -0600 Subject: [PATCH 170/477] wet_scav - Fixing compilation errors. --- .../src/physics/mam/eamxx_mam_wetscav_process_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index bbd99a66cfb0..b598fa6d7483 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -463,9 +463,9 @@ void MAMWetscav::run_impl(const double dt) { // Zero out tendencies otherwise, they are initialized to junk values for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - Kokkos::deep_copy(tends.n_mode_i[m], 0); + Kokkos::deep_copy(dry_aero_tends_.int_aero_nmr[m], 0); for(int a = 0; a < mam4::num_species_mode(m); ++a) { - Kokkos::deep_copy(tends.q_aero_i[m][a], 0); + Kokkos::deep_copy(dry_aero_tends_.int_aero_mmr[m][a], 0); } } From d3a1ebc9f670db2c74fbf150183423f4bec173a7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 15 Jul 2024 22:19:19 -0700 Subject: [PATCH 171/477] More prints for validation --- .../eamxx_mam_dry_deposition_functions.hpp | 51 ++++++++++- ...x_mam_dry_deposition_process_interface.cpp | 86 +++++++++++++++++-- ...x_mam_dry_deposition_process_interface.hpp | 2 +- externals/mam4xx | 2 +- 4 files changed, 128 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index ff876cdcf425..034471f047ea 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -30,7 +30,7 @@ void compute_tendencies( MAMDryDep::view_2d qqcw_[mam4::aero_model::pcnst], // outputs - MAMDryDep::view_3d d_qtracers_dt, MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_3d ptend_q, MAMDryDep::view_2d aerdepdrycw, MAMDryDep::view_2d aerdepdryis, // work arrays @@ -43,7 +43,7 @@ void compute_tendencies( [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { auto printb = [](const std::string &name, const double &val) { - std::cout << name << ":" << std::setprecision(15) << val << std::endl; + // std::cout << name << ":" << std::setprecision(15) << val << std::endl; }; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); // FIXME: WHY we are using a new policy here?? can't we get it from the @@ -140,14 +140,59 @@ void compute_tendencies( // input-outputs qqcw, // outputs - ekat::subview(d_qtracers_dt, icol), ptend_lq, + ekat::subview(ptend_q, icol), ptend_lq, ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), // work arrays rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); printb("aft:qqcw:", qqcw[23](63)); + printb("aft:ptend:", ptend_q(0, 63, 23)); }); // parallel_for for ncols } +void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, + const int ncol, const int nlev, + // output + const mam_coupling::AerosolState dry_aero) { + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + 1, nlev); + static constexpr int nmodes = mam4::AeroConfig::num_modes(); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + for(int m = 0; m < nmodes; ++m) { + dry_aero.int_aero_nmr[m](icol, kk) += + ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + dry_aero.int_aero_mmr[m][a](icol, kk) += + ptend_q(icol, kk, + mam4::ConvProc::lmassptrcw_amode(a, m)) * + dt; + } + }); // parallel_for nlevs + }); +} + +void update_cloudborne_mmrs( + const MAMDryDep::view_2d qqcw[mam4::aero_model::pcnst], const double dt, + const int ncol, const int nlev_, + // output + const mam_coupling::AerosolState dry_aero) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], + qqcw[mam4::ConvProc::numptrcw_amode(m)]); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], + qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)]); + } + } + } +} + void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, 0.16803558403621365E-001, 0.18076055155371872E-001, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index a38ad080a3d4..a40845b8c0b1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -361,8 +361,8 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // FIXME: comment what they are and units..... qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - rho_ = view_2d("rho", ncol_, nlev_); - d_qtracers_dt_ = view_3d("d_qtracers_dt_", ncol_, nlev_, pcnst); + rho_ = view_2d("rho", ncol_, nlev_); + ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { for(int j = 0; j < aerosol_categories_; ++j) { @@ -388,12 +388,16 @@ void MAMDryDep::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, - dry_aero_); + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // initialize_impl // ========================================================================================= void MAMDryDep::run_impl(const double dt) { + auto printb = [](const std::string &name, const double &val) { + std::cout << name << ":" << std::setprecision(15) << val << std::endl; + }; + using DryDep = mam4::DryDeposition; const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); @@ -441,7 +445,33 @@ void MAMDryDep::run_impl(const double dt) { .get_view(); populated_fraction_landuse(fraction_landuse_, ncol_); - // std::cout<<"fraction_landuse_:"< 0) { + printb("inter_before:", dry_aero_.int_aero_mmr[m][a](0, 63)); + } + } + } + + for(int m = 0; m < num_aero_modes; ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + printb("cld_before:", dry_aero_.cld_aero_nmr[m](0, 63)); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + printb("cld_before:", dry_aero_.cld_aero_mmr[m][a](0, 63)); + } + } + } + compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, @@ -450,13 +480,53 @@ void MAMDryDep::run_impl(const double dt) { // Inouts-outputs qqcw_, // Outputs - d_qtracers_dt_, aerdepdrycw_, aerdepdryis_, + ptend_q_, aerdepdrycw_, aerdepdryis_, // work arrays rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); Kokkos::fence(); + + // Update the interstitial aerosols using ptend. + update_interstitial_mmrs(ptend_q_, dt, ncol_, nlev_, // inputs + dry_aero_); // output + + // Update the interstitial aerosols + update_cloudborne_mmrs(qqcw_, dt, ncol_, nlev_, // inputs + dry_aero_); // output + + for(int m = 0; m < num_aero_modes; ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + printb("inter_after:", dry_aero_.int_aero_nmr[m](0, 63)); + std::cout << int_nmr_field_name << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + printb("inter_after:", dry_aero_.int_aero_mmr[m][a](0, 63)); + std::cout << int_mmr_field_name << std::endl; + } + } + } + + for(int m = 0; m < num_aero_modes; ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + printb("cld_after:", dry_aero_.cld_aero_nmr[m](0, 63)); + std::cout << cld_nmr_field_name << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + printb("cld_after:", dry_aero_.cld_aero_mmr[m][a](0, 63)); + std::cout << cld_mmr_field_name << std::endl; + } + } + } + // call post processing to convert dry mixing ratios to wet mixing ratios // and update the state - // Kokkos::parallel_for("postprocess", scan_policy, postprocess_); - // Kokkos::fence(); // wait before returning to calling function + Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + Kokkos::fence(); // wait before returning to calling function } // run_impl } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 867a81d9fb40..8a28b940ee15 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -45,7 +45,7 @@ class MAMDryDep final : public scream::AtmosphereProcess { std::shared_ptr grid_; view_3d qtracers_; - view_3d d_qtracers_dt_; + view_3d ptend_q_; // inputs // FIXME: collect all inputs and outputs together diff --git a/externals/mam4xx b/externals/mam4xx index 535f67b3dd20..89fd96c4a81f 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 535f67b3dd20e5df714149eeefdb7b50ba9862ed +Subproject commit 89fd96c4a81f9b22b0f24fc35282a04599443efc From 123ec08857e58715e6d8b8f8bca47d4ba9540b4f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 16 Jul 2024 06:51:26 -0700 Subject: [PATCH 172/477] Cleaned some prints and adds new comments --- .../eamxx_mam_dry_deposition_functions.hpp | 42 +++++++------- ...x_mam_dry_deposition_process_interface.cpp | 58 +++---------------- externals/mam4xx | 2 +- 3 files changed, 29 insertions(+), 73 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 034471f047ea..d7d1542eeced 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -42,22 +42,20 @@ void compute_tendencies( MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] [MAMDryDep::aerosol_categories_], MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { - auto printb = [](const std::string &name, const double &val) { - // std::cout << name << ":" << std::setprecision(15) << val << std::endl; - }; static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); - // FIXME: WHY we are using a new policy here?? can't we get it from the - // run_impl?? const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( - 1, nlev); + 1, nlev); // FIXME: change 1 to ncol + + // Parallel loop over all the columns Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { static constexpr int num_aero_species = mam_coupling::num_aero_species(); const int icol = team.league_rank(); - + // Parallel loop over all the levels to populate qtracers array using + // dry_aero Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { for(int mode = 0; mode < num_aero_modes; ++mode) { @@ -75,8 +73,11 @@ void compute_tendencies( }); // parallel_for for nlevs team.team_barrier(); + // Create atm and progs objects mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + + // Extract column data (or 1d view) from 2d views of data mam4::ConstColumnView dgncur_awet[num_aero_modes]; mam4::ConstColumnView wet_dens[num_aero_modes]; @@ -94,11 +95,9 @@ void compute_tendencies( fraction_landuse[i] = fraction_landuse_[i](icol); } - // FIXME: why mam4::ColumnView didn;t work here, why use - // Kokkos::View. Solution: Use ColumnView in drydep.hpp as well. static constexpr int nmodes = mam4::AeroConfig::num_modes(); - mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_], - vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; + mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_]; + mam4::ColumnView vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; Real vlc_trb[nmodes][MAMDryDep::aerosol_categories_]; @@ -116,7 +115,7 @@ void compute_tendencies( qqcw[i] = ekat::subview(qqcw_[i], icol); dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); } - // Extract Prognostics + // Extract qqcw from Prognostics Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { for(int m = 0; m < nmodes; ++m) { @@ -128,8 +127,8 @@ void compute_tendencies( progs.q_aero_c[m][a][kk]; } }); // parallel_for nlevs - bool ptend_lq[pcnst]; - printb("bef:qqcw:", qqcw[23](63)); + + bool ptend_lq[pcnst]; // currently unused mam4::aero_model_drydep( // inputs team, fraction_landuse, atm.temperature, atm.pressure, @@ -144,11 +143,10 @@ void compute_tendencies( ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), // work arrays rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); - printb("aft:qqcw:", qqcw[23](63)); - printb("aft:ptend:", ptend_q(0, 63, 23)); }); // parallel_for for ncols -} +} // Compute_tendencies ends +// Update interstitial aerosols using ptend_q tendencies void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, const int ncol, const int nlev, // output @@ -173,12 +171,13 @@ void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, dt; } }); // parallel_for nlevs - }); -} + }); // parallel_for icol +} // Update interstitial aerosols ends +// Update cloud borne aerosols using qqcw void update_cloudborne_mmrs( const MAMDryDep::view_2d qqcw[mam4::aero_model::pcnst], const double dt, - const int ncol, const int nlev_, + const int nlev_, // output const mam_coupling::AerosolState dry_aero) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -191,8 +190,9 @@ void update_cloudborne_mmrs( } } } -} +} // Update cloud borne aerosols ends +// FIXME: remove the following function void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, 0.16803558403621365E-001, 0.18076055155371872E-001, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index a40845b8c0b1..7599324db1a2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -3,57 +3,13 @@ // Drydep functions are stored in the following hpp file #include -#include "mam4xx/drydep.hpp" //FIXME: Do we need it here??? - /* -Inputs: - -Atmosphere: - temperature "T_mid" - pressure "p_mid" - interface_pressure "p_int" - hydrostatic_dp "pseudo_density" - -Diagnostics: - tracer_mixing_ratio "qtracers" - wet_geometric_mean_diameter_i "dgncur_awet" - wet_density "wetdens" - -Diagnostic Scalar Parameters, one per column: - "Obukhov_length" - "surface_friction_velocty" - "land_fraction" - "ice_fraction" - "ocean_fraction" - "friction_velocity" - "aerodynamical_resistance" - -Prognostics: - n_mode_c "mam_coupling::cld_aero_nmr_field_name(m)" - q_aero_c "mam_coupling::int_aero_nmr_field_name(m)" - - - -Outputs: - -Diagnostics: - d_tracer_mixing_ratio_dt "d_qtracers_dt" - deposition_flux_of_cloud_borne_aerosols -"deposition_flux_of_cloud_borne_aerosols" - deposition_flux_of_interstitial_aerosols -"deposition_flux_of_interstitial_aerosols" - -Computed internally, could be exposed if needed by another process - vlc_grv(nlev) : dep velocity of gravitational settling [m/s] - vlc_trb(nlev) : dep velocity of turbulent dry deposition [m/s] - vlc_dry(nlev) : dep velocity, sum of vlc_grv and vlc_trb [m/s] - -Tendencies: - n_mode_c "Tendencies" - q_aero_c "Tendencies" - +----------------------------------------------------------------- +NOTES: +1. Add a CIME test and multi-process tests +2. Ensure that the submodule for MAM4xx is the main branch +----------------------------------------------------------------- */ - namespace scream { MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) @@ -490,8 +446,8 @@ void MAMDryDep::run_impl(const double dt) { dry_aero_); // output // Update the interstitial aerosols - update_cloudborne_mmrs(qqcw_, dt, ncol_, nlev_, // inputs - dry_aero_); // output + update_cloudborne_mmrs(qqcw_, dt, nlev_, // inputs + dry_aero_); // output for(int m = 0; m < num_aero_modes; ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); diff --git a/externals/mam4xx b/externals/mam4xx index 89fd96c4a81f..5e7ff7200b49 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 89fd96c4a81f9b22b0f24fc35282a04599443efc +Subproject commit 5e7ff7200b4910a43fbd254951c6ff61b6463e30 From a5aec91cf445c5314bd476cb5e192a04a7f4c92b Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 16 Jul 2024 10:28:54 -0700 Subject: [PATCH 173/477] change ekat::impl::max instances to std::max and remove unneeded m_helper_fields line --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 42ea7d0ec03f..cf20435fd2ce 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - ekat::impl::max(m_params.get("target_latitude"),0.1); + std::max(m_params.get("target_latitude"),0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - ekat::impl::max(m_params.get("target_longitude"),0.1); + std::max(m_params.get("target_longitude"),0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -548,7 +548,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) // Convert to pressure to millibar (file gives pressure in Pa) for (int ilev=0; ilev Date: Tue, 16 Jul 2024 16:34:51 -0700 Subject: [PATCH 174/477] create a general compset that is tested with the dycoms case, an ARM97 compset and test, and add test case for RCE --- cime_config/tests.py | 5 ++- .../eamxx/cime_config/config_component.xml | 2 +- .../eamxx/cime_config/config_compsets.xml | 45 +++++++++++-------- .../cime_config/namelist_defaults_scream.xml | 36 ++++++++++----- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 06c16ca176c9..275754e26a40 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -655,8 +655,11 @@ "e3sm_scream_v1_dp-eamxx" : { "time" : "01:00:00", + # each test 225 phys cols, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-DYCOMSrf01", # 225 phys cols, roughly size of ne2 + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-ARM97", + "ERS_P16_Ln22.ne30_ne30.F2000-SCREAMv1-DP-RCE", ) }, diff --git a/components/eamxx/cime_config/config_component.xml b/components/eamxx/cime_config/config_component.xml index d3b6e34744d0..7380b371401d 100644 --- a/components/eamxx/cime_config/config_component.xml +++ b/components/eamxx/cime_config/config_component.xml @@ -60,7 +60,7 @@ - + char diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index d33b865bde93..504c8dfbf848 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -75,19 +75,19 @@ - F2010-SCREAMv1-DP-DYCOMSrf01 - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%DYCOMSrf01 + F2010-SCREAMv1-DP-General + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%General Experimental, under development - F2010-SCREAMv1-DP - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx + F2010-SCREAMv1-DP-ARM97 + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%ARM97 Experimental, under development - F2000-SCREAMv1-RCE-DP + F2000-SCREAMv1-DP-RCE 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx Experimental, under development @@ -98,11 +98,13 @@ 2016-08-01 2020-01-20 - 1999-07-10 + 1999-07-10 + 1999-06-23 + 2000-01-01 - + 864 @@ -121,39 +123,44 @@ - + - 31.5 + 225 - + - 238.5 + 1 - + - 225 + 225 - + - 1 + 1 - + + - 225 + 31.5 + 36.605 + 0.0 - + - 1 + 238.5 + 262.515 + 0.0 diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index bc09ce40952d..3a0722220b2d 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -578,25 +578,41 @@ be lost if SCREAM_HACK_XML is not enabled. ${CASE} - + true UNSET - ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -999 - 31.5 -999 - 238.5 false - true false - true false false 1100 0 10800 false + + + + ${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc + 36.605 + 262.515 + true + true + + + ${DIN_LOC_ROOT}/atm/cam/scam/iop/RCE_iopfile_4scam.nc + 0.0 + 0.0 + + + ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc + 31.5 + 238.5 + true + true + @@ -649,13 +665,13 @@ be lost if SCREAM_HACK_XML is not enabled. 1024 0 0 - 5 + 5 0 - 5 + 5 0 - 50000 + 50000 0 - 50000 + 50000 -1 4 cube From 9f3d4930c75a439e3a2ceecfbc5b5a3b707ad706 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 16 Jul 2024 16:44:29 -0700 Subject: [PATCH 175/477] update some comments to make them more clear --- components/eamxx/cime_config/config_compsets.xml | 2 +- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index 504c8dfbf848..77c2de92ed32 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -104,7 +104,7 @@ - + 864 diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 3a0722220b2d..f1264b8b4e08 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -593,7 +593,7 @@ be lost if SCREAM_HACK_XML is not enabled. 10800 false - + ${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc 36.605 From 8b9b3f01684235591fdccdae385db58cd168d98a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 17 Jul 2024 06:58:00 -0700 Subject: [PATCH 176/477] Adds a CIME test and fixed FIXME comments and removed prinouts --- .../cime_config/namelist_defaults_scream.xml | 4 + .../scream/mam4xx/drydep/shell_commands | 13 +++ .../eamxx_mam_dry_deposition_functions.hpp | 6 +- ...x_mam_dry_deposition_process_interface.cpp | 92 ++++--------------- 4 files changed, 39 insertions(+), 76 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50188236e4e0..e586c6ce2151 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -241,6 +241,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0 + + + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands new file mode 100644 index 000000000000..d219b2a489e0 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands @@ -0,0 +1,13 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +#modify initial condition file to get aerosol species ICs +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b + +# Add drydep process +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_drydep" -b + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index d7d1542eeced..98a59afab79e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,7 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, // FIXME: Make it a constant view + MAMDryDep::view_3d qtracers, MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], const MAMDryDep::const_view_3d dgncur_awet_, const MAMDryDep::const_view_3d wet_dens_, @@ -45,7 +45,7 @@ void compute_tendencies( static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( - 1, nlev); // FIXME: change 1 to ncol + ncol, nlev); // Parallel loop over all the columns Kokkos::parallel_for( @@ -192,7 +192,7 @@ void update_cloudborne_mmrs( } } // Update cloud borne aerosols ends -// FIXME: remove the following function +// FIXME: remove the following function after implementing file read for landuse void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, 0.16803558403621365E-001, 0.18076055155371872E-001, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 7599324db1a2..c7f2471c447f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -8,6 +8,7 @@ NOTES: 1. Add a CIME test and multi-process tests 2. Ensure that the submodule for MAM4xx is the main branch +3. Read file for fractional landuse ----------------------------------------------------------------- */ namespace scream { @@ -56,7 +57,7 @@ void MAMDryDep::set_grids( }; const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"ncol", "num_modes", "lev"}); + make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "num_modes", "lev"}); using namespace ekat::units; @@ -243,16 +244,20 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Populate the wet atmosphere state with views from fields // FIMXE: specifically look which among these are actually used by the process wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS wet_atm_.qc = get_field_in("qc").get_view(); wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); // Populate the dry atmosphere state with views from fields - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); @@ -322,21 +327,21 @@ void MAMDryDep::initialize_impl(const RunType run_type) { for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { for(int j = 0; j < aerosol_categories_; ++j) { - Kokkos::resize(vlc_dry_[i][j], ncol_, nlev_); - Kokkos::resize(vlc_grv_[i][j], ncol_, nlev_); - Kokkos::resize(vlc_trb_[i][j], ncol_); + vlc_dry_[i][j] = view_2d("vlc_dry_[i][j]", ncol_, nlev_); + vlc_grv_[i][j] = view_2d("vlc_grv_[i][j]", ncol_, nlev_); + vlc_trb_[i][j] = view_1d("vlc_trb_[i][j]", ncol_); } } for(int i = 0; i < pcnst; ++i) { - Kokkos::resize(qqcw_[i], ncol_, nlev_); - Kokkos::resize(dqdt_tmp_[i], ncol_, nlev_); + qqcw_[i] = view_2d("qqcw_[i]", ncol_, nlev_); + dqdt_tmp_[i] = view_2d("dqdt_tmp_[i]", ncol_, nlev_); } static constexpr int n_land_type = mam4::DryDeposition::n_land_type; for(int i = 0; i < n_land_type; ++i) { // FIXME: This should come from a file reading - Kokkos::resize(fraction_landuse_[i], ncol_); + fraction_landuse_[i] = view_1d("fraction_landuse_[i]", ncol_); } //----------------------------------------------------------------- @@ -350,13 +355,9 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - auto printb = [](const std::string &name, const double &val) { - std::cout << name << ":" << std::setprecision(15) << val << std::endl; - }; - using DryDep = mam4::DryDeposition; const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -400,34 +401,10 @@ void MAMDryDep::run_impl(const double dt) { auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); + //FIXME: remove it if it read from a file populated_fraction_landuse(fraction_landuse_, ncol_); - for(int m = 0; m < num_aero_modes; ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - printb("inter_before:", dry_aero_.int_aero_nmr[m](0, 63)); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - - if(strlen(int_mmr_field_name) > 0) { - printb("inter_before:", dry_aero_.int_aero_mmr[m][a](0, 63)); - } - } - } - - for(int m = 0; m < num_aero_modes; ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - printb("cld_before:", dry_aero_.cld_aero_nmr[m](0, 63)); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - - if(strlen(cld_mmr_field_name) > 0) { - printb("cld_before:", dry_aero_.cld_aero_mmr[m][a](0, 63)); - } - } - } + //Call drydeposition and get tendencies compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, @@ -449,37 +426,6 @@ void MAMDryDep::run_impl(const double dt) { update_cloudborne_mmrs(qqcw_, dt, nlev_, // inputs dry_aero_); // output - for(int m = 0; m < num_aero_modes; ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - printb("inter_after:", dry_aero_.int_aero_nmr[m](0, 63)); - std::cout << int_nmr_field_name << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - - if(strlen(int_mmr_field_name) > 0) { - printb("inter_after:", dry_aero_.int_aero_mmr[m][a](0, 63)); - std::cout << int_mmr_field_name << std::endl; - } - } - } - - for(int m = 0; m < num_aero_modes; ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - printb("cld_after:", dry_aero_.cld_aero_nmr[m](0, 63)); - std::cout << cld_nmr_field_name << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - - if(strlen(cld_mmr_field_name) > 0) { - printb("cld_after:", dry_aero_.cld_aero_mmr[m][a](0, 63)); - std::cout << cld_mmr_field_name << std::endl; - } - } - } - // call post processing to convert dry mixing ratios to wet mixing ratios // and update the state Kokkos::parallel_for("postprocess", scan_policy, postprocess_); From f2a7b976fce6f5750518773349f0f4351f227796 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 17 Jul 2024 09:42:48 -0700 Subject: [PATCH 177/477] cast minimum lat/lon threshold as a real --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index cf20435fd2ce..6f8474f8d85e 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),0.1); + std::max(m_params.get("target_latitude"),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - std::max(m_params.get("target_longitude"),0.1); + std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), From a4a5798d449547de67b1414b12541b8f2772e971 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 17 Jul 2024 12:12:26 -0700 Subject: [PATCH 178/477] Fixes single process test and some merge master fixes --- .../eamxx/src/physics/mam/CMakeLists.txt | 1 - ...x_mam_dry_deposition_process_interface.cpp | 9 ++- .../mam/impl/compute_water_content.cpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 33 ++++------- .../eamxx/src/share/field/field_tag.hpp | 43 -------------- components/eamxx/tests/CMakeLists.txt | 1 - .../single-process/mam/drydep/CMakeLists.txt | 26 +++++++- .../single-process/mam/drydep/input.yaml | 11 +++- .../single-process/mam/drydep/output.yaml | 59 ++++++++++++++++++- 9 files changed, 107 insertions(+), 78 deletions(-) diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 6339c85aef0b..e41517077538 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -45,7 +45,6 @@ add_library(mam eamxx_mam_optics_process_interface.cpp eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp) - target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index c7f2471c447f..ebbab7ab7571 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -145,7 +145,7 @@ void MAMDryDep::set_grids( //----------- Variables from other mam4xx processes ------------ // Geometric mean wet diameter for number distribution [m] - add_field("dgncur_awet", scalar4d_mid, m, grid_name); + add_field("dgnumwet", scalar4d_mid, m, grid_name); // Wet density of interstitial aerosol [kg/m3] add_field("wetdens", scalar4d_mid, kg / m3, grid_name); @@ -355,7 +355,6 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMDryDep::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -368,7 +367,7 @@ void MAMDryDep::run_impl(const double dt) { // ------------------------------------------------------------- // Geometric mean wet diameter for number distribution [m] - auto dgncur_awet_ = get_field_in("dgncur_awet").get_view(); + auto dgncur_awet_ = get_field_in("dgnumwet").get_view(); // Wet density of interstitial aerosol [kg/m3] auto wet_dens_ = get_field_in("wetdens").get_view(); // Obukhov length [m] @@ -401,10 +400,10 @@ void MAMDryDep::run_impl(const double dt) { auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") .get_view(); - //FIXME: remove it if it read from a file + // FIXME: remove it if it read from a file populated_fraction_landuse(fraction_landuse_, ncol_); - //Call drydeposition and get tendencies + // Call drydeposition and get tendencies compute_tendencies(ncol_, nlev_, dt, obukhov_length_, surface_friction_velocty_, land_fraction_, ice_fraction_, ocean_fraction_, friction_velocity_, diff --git a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp index 673fe06292e1..ea7190afff91 100644 --- a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp +++ b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp @@ -23,7 +23,7 @@ void compute_water_content(const mam4::Prognostics &progs, int k, // extract aerosol tracers for this level into state_q, which is needed // for computing dry aerosol properties below // FIXME: we should eliminate this index translation stuff - constexpr int nvars = mam4::aero_model::pcnst; + constexpr int nvars = aero_model::pcnst; Real state_q[nvars]; // aerosol tracers for level k for (int imode = 0; imode < num_modes; ++imode) { int la, lc; // interstitial and cloudborne indices within state_q diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 8e77ae2484b8..1e13cece98d9 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -251,10 +251,7 @@ const char* cld_aero_mmr_field_name(const int mode, const int species) { // mixing ratio field in EAMxx KOKKOS_INLINE_FUNCTION const char* gas_mmr_field_name(const int gas) { - if (!gas_mmr_names(gas)[0]) { - concat_2_strings("", gas_species_name(gas), gas_mmr_names(gas)); - } - return gas_mmr_names(gas); + return const_cast(gas_species_name(gas)); } // This type stores multi-column views related specifically to the wet @@ -351,9 +348,8 @@ struct Buffer { // ======================= // number of local fields stored at column interfaces - static constexpr int num_2d_iface = 2; + static constexpr int num_2d_iface = 1; - uview_2d p_int; // pressure at interfaces uview_2d z_iface; // height at interfaces // storage @@ -469,7 +465,6 @@ inline size_t init_buffer(const ATMBufferManager &buffer_manager, // set view pointers for interface fields uview_2d* view_2d_iface_ptrs[Buffer::num_2d_iface] = { - &buffer.p_int, &buffer.z_iface }; for (int i = 0; i < Buffer::num_2d_iface; ++i) { @@ -524,8 +519,6 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, "cldfrac not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, "w_updraft not defined for dry atmosphere state!"); - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_int.data() != nullptr, - "p_int not defined for dry atmosphere state!"); return haero::Atmosphere(mam4::nlev, ekat::subview(dry_atm.T_mid, column_index), ekat::subview(dry_atm.p_mid, column_index), @@ -656,11 +649,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -681,21 +674,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);// PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } @@ -766,7 +759,7 @@ void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs const auto PC = mam4::ModeIndex::PrimaryCarbon; \ const auto NoMode = mam4::ModeIndex::None; \ static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index e500250b80f2..673208d6ae86 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -34,15 +34,6 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, - // Added for RRTMGP, TODO: Revisit this approach, is there a better way than adding more field tags? - Gases, - ShortWaveBand, - ShortWaveGpoint, - LongWaveBand, - LongWaveGpoint, - IsccpTau, - IsccpPrs, - num_modes }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -59,15 +50,6 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; - // Added for rrtmgp - see TODO item above - constexpr auto NGAS = FieldTag::Gases; - constexpr auto SWBND = FieldTag::ShortWaveBand; - constexpr auto LWBND = FieldTag::LongWaveBand; - constexpr auto SWGPT = FieldTag::ShortWaveGpoint; - constexpr auto LWGPT = FieldTag::LongWaveGpoint; - constexpr auto ISCCPTAU = FieldTag::IsccpTau; - constexpr auto ISCCPPRS = FieldTag::IsccpPrs; - constexpr auto NMODES = FieldTag::num_modes; } inline std::string e2str (const FieldTag ft) { @@ -98,31 +80,6 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; - // Added for rrtmgp - see TODO item above - case FieldTag::Gases: - name = "ngas"; - break; - case FieldTag::ShortWaveBand: - name = "swband"; - break; - case FieldTag::ShortWaveGpoint: - name = "swgpt"; - break; - case FieldTag::LongWaveBand: - name = "lwband"; - break; - case FieldTag::LongWaveGpoint: - name = "lwgpt"; - break; - case FieldTag::IsccpTau: - name = "ISCCPTAU"; - break; - case FieldTag::IsccpPrs: - name = "ISCCPPRS"; - break; - case FieldTag::num_modes: - name = "num_modes"; - break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 635f5de80a87..5fd23a41d9d8 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -63,7 +63,6 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") - set(EAMxx_tests_IC_FILE_MAM "scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc") set(EAMxx_tests_IC_FILE_MAM4xx_72lev "screami_unit_tests_mam4xx_ne2np4L72_20240329.nc") # Testing individual atm processes diff --git a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt index 698b9cb7b542..d05e6823642f 100644 --- a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt @@ -1,18 +1,19 @@ include (ScreamUtils) set (TEST_BASE_NAME mam4_drydep_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LABELS mam4_drydep physics LIBS mam MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) # Set AD configurable options -set (ATM_TIME_STEP 3600) -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h set (RUN_T0 2021-10-12-45000) ## Copy (and configure) yaml files needed by tests @@ -24,3 +25,22 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_drydep physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() + diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index a95faf9047a4..1de57ad7e682 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -23,11 +23,20 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /qfs/people/sing201/eagles/refactor_mam/mam4xx_scream/3src/screami_unit_tests_mam4xx_ne2np4L72_LAT_m38.0068938875434_LON_16p9323465700125_DRYDEP.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} fraction_landuse: 0.2 w_updraft: 1e-5 pbl_height: 0.0 + Obukhov_length: -0.48167567011091336E+002 + friction_velocity: 0.47569674433601039E+000 + land_fraction: 0.14675684171817760E+000 + ice_fraction: 0.00000000000000000E+000 + ocean_fraction: 0.85324315828182240E+000 + surface_friction_velocty: 0.27482164092221828E+000 + aerodynamical_resistance: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/single-process/mam/drydep/output.yaml b/components/eamxx/tests/single-process/mam/drydep/output.yaml index e900bb804340..6d8a79e8fe00 100644 --- a/components/eamxx/tests/single-process/mam/drydep/output.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/output.yaml @@ -2,10 +2,63 @@ --- filename_prefix: mam4_drydep_standalone_output Averaging Type: Instant -Field Names: - - T_mid +Fields: + Physics: + Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols + output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... From f508702d342902a4e9faeaf5611b618f71e765e1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 17 Jul 2024 12:19:22 -0700 Subject: [PATCH 179/477] Removes commented-out scorpio error message code --- components/eamxx/src/share/io/scream_scorpio_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 68a432819ec3..cb2b501f2b5c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -360,11 +360,11 @@ void finalize_subsystem () EKAT_REQUIRE_MSG (s.pio_sysid!=-1, "Error! PIO subsystem was already finalized.\n"); - /*for (auto& it : s.files) { + for (auto& it : s.files) { EKAT_REQUIRE_MSG (it.second.num_customers==0, "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" " - filename: " + it.first + "\n"); - }*/ + } s.files.clear(); for (auto& it : s.decomps) { From 9e4978bb3ccc4bf0353109c8671430ba0a3c1d39 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 18 Jul 2024 10:18:19 -0600 Subject: [PATCH 180/477] Fix GPU tests: Do not use host pointers on device. --- .../eamxx_mam_dry_deposition_functions.hpp | 68 +++++++++---------- ...x_mam_dry_deposition_process_interface.cpp | 22 ++---- ...x_mam_dry_deposition_process_interface.hpp | 23 ++++--- 3 files changed, 52 insertions(+), 61 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 98a59afab79e..1fdd8fd1243e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -20,14 +20,14 @@ void compute_tendencies( const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, MAMDryDep::view_3d qtracers, - MAMDryDep::view_1d fraction_landuse_[MAMDryDep::n_land_type], + MAMDryDep::view_2d fraction_landuse_, const MAMDryDep::const_view_3d dgncur_awet_, const MAMDryDep::const_view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, const mam_coupling::AerosolState dry_aero, // input-outputs - MAMDryDep::view_2d qqcw_[mam4::aero_model::pcnst], + MAMDryDep::view_3d qqcw_, // outputs MAMDryDep::view_3d ptend_q, MAMDryDep::view_2d aerdepdrycw, @@ -35,13 +35,10 @@ void compute_tendencies( // work arrays MAMDryDep::view_2d rho_, - MAMDryDep::view_2d vlc_dry_[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - MAMDryDep::view_1d vlc_trb_[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - MAMDryDep::view_2d vlc_grv_[mam4::AeroConfig::num_modes()] - [MAMDryDep::aerosol_categories_], - MAMDryDep::view_2d dqdt_tmp_[mam4::aero_model::pcnst]) { + MAMDryDep::view_4d vlc_dry_, + MAMDryDep::view_3d vlc_trb_, + MAMDryDep::view_4d vlc_grv_, + MAMDryDep::view_3d dqdt_tmp_) { static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( @@ -92,32 +89,31 @@ void compute_tendencies( static constexpr int n_land_type = MAMDryDep::n_land_type; Real fraction_landuse[n_land_type]; for(int i = 0; i < n_land_type; ++i) { - fraction_landuse[i] = fraction_landuse_[i](icol); + fraction_landuse[i] = fraction_landuse_(i, icol); } static constexpr int nmodes = mam4::AeroConfig::num_modes(); mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_]; mam4::ColumnView vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; - Real vlc_trb[nmodes][MAMDryDep::aerosol_categories_]; for(int i = 0; i < nmodes; ++i) { for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { - vlc_dry[i][j] = ekat::subview(vlc_dry_[i][j], icol); - vlc_trb[i][j] = vlc_trb_[i][j](icol); - vlc_grv[i][j] = ekat::subview(vlc_grv_[i][j], icol); + vlc_dry[i][j] = ekat::subview(vlc_dry_, i, j, icol); + vlc_trb[i][j] = vlc_trb_(i, j, icol); + vlc_grv[i][j] = ekat::subview(vlc_grv_, i, j, icol); } } static constexpr int pcnst = mam4::aero_model::pcnst; mam4::ColumnView qqcw[pcnst]; mam4::ColumnView dqdt_tmp[pcnst]; for(int i = 0; i < pcnst; ++i) { - qqcw[i] = ekat::subview(qqcw_[i], icol); - dqdt_tmp[i] = ekat::subview(dqdt_tmp_[i], icol); + qqcw[i] = ekat::subview(qqcw_, i, icol); + dqdt_tmp[i] = ekat::subview(dqdt_tmp_, i, icol); } // Extract qqcw from Prognostics Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, nlev), [&](int kk) { for(int m = 0; m < nmodes; ++m) { qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; @@ -127,7 +123,7 @@ void compute_tendencies( progs.q_aero_c[m][a][kk]; } }); // parallel_for nlevs - + team.team_barrier(); bool ptend_lq[pcnst]; // currently unused mam4::aero_model_drydep( // inputs @@ -159,7 +155,7 @@ void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { const int icol = team.league_rank(); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, nlev), [&](int kk) { for(int m = 0; m < nmodes; ++m) { dry_aero.int_aero_nmr[m](icol, kk) += ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; @@ -176,36 +172,38 @@ void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, // Update cloud borne aerosols using qqcw void update_cloudborne_mmrs( - const MAMDryDep::view_2d qqcw[mam4::aero_model::pcnst], const double dt, + const MAMDryDep::view_3d qqcw, const double dt, const int nlev_, // output const mam_coupling::AerosolState dry_aero) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], - qqcw[mam4::ConvProc::numptrcw_amode(m)]); + ekat::subview(qqcw, mam4::ConvProc::numptrcw_amode(m))); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { if(dry_aero.cld_aero_mmr[m][a].data()) { Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], - qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)]); + ekat::subview(qqcw, mam4::ConvProc::lmassptrcw_amode(a, m))); } } } } // Update cloud borne aerosols ends // FIXME: remove the following function after implementing file read for landuse -void populated_fraction_landuse(MAMDryDep::view_1d flu[11], const int ncol) { - Real temp[11] = {0.28044346587077795E-003, 0.26634987180780171E-001, - 0.16803558403621365E-001, 0.18076055155371872E-001, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.91803784897907303E+000, 0.17186036997038400E-002, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.18448503115578840E-001}; - - for(int icol = 0; icol < ncol; ++icol) { - for(int kk = 0; kk < 11; ++kk) { - flu[kk](icol) = temp[kk]; - } - } +void populated_fraction_landuse(MAMDryDep::view_2d flu, const int ncol) { + Kokkos::parallel_for("populated_fraction_landuse", 1, KOKKOS_LAMBDA(int ) { + static constexpr int n_land_type = MAMDryDep::n_land_type; + const Real temp[n_land_type] = + {0.28044346587077795E-003, 0.26634987180780171E-001, + 0.16803558403621365E-001, 0.18076055155371872E-001, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.91803784897907303E+000, 0.17186036997038400E-002, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.18448503115578840E-001}; + for(int i = 0; i < n_land_type; ++i) + for(int j = 0; j < ncol; ++j) + flu(i, j) = temp[i]; + }); + Kokkos::fence(); } } // namespace diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index ebbab7ab7571..b4625387352f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -325,24 +325,16 @@ void MAMDryDep::initialize_impl(const RunType run_type) { rho_ = view_2d("rho", ncol_, nlev_); ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); - for(int i = 0; i < mam4::AeroConfig::num_modes(); ++i) { - for(int j = 0; j < aerosol_categories_; ++j) { - vlc_dry_[i][j] = view_2d("vlc_dry_[i][j]", ncol_, nlev_); - vlc_grv_[i][j] = view_2d("vlc_grv_[i][j]", ncol_, nlev_); - vlc_trb_[i][j] = view_1d("vlc_trb_[i][j]", ncol_); - } - } + vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_, nlev_); + vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_, nlev_); + vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_); - for(int i = 0; i < pcnst; ++i) { - qqcw_[i] = view_2d("qqcw_[i]", ncol_, nlev_); - dqdt_tmp_[i] = view_2d("dqdt_tmp_[i]", ncol_, nlev_); - } + qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); + dqdt_tmp_ = view_3d("dqdt_tmp_", pcnst, ncol_, nlev_); static constexpr int n_land_type = mam4::DryDeposition::n_land_type; - for(int i = 0; i < n_land_type; ++i) { - // FIXME: This should come from a file reading - fraction_landuse_[i] = view_1d("fraction_landuse_[i]", ncol_); - } + // FIXME: This should come from a file reading + fraction_landuse_ = view_2d("fraction_landuse_", n_land_type, ncol_); //----------------------------------------------------------------- // Setup preprocessing and post processing diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 8a28b940ee15..8feb71e04693 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -16,17 +16,18 @@ namespace scream { // stores exactly ONE instance of this class in its list of subcomponents. class MAMDryDep final : public scream::AtmosphereProcess { public: + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int aerosol_categories_ = + mam4::DryDeposition::aerosol_categories; + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; + using view_1d = Field::view_dev_t; using view_2d = Field::view_dev_t; using view_3d = Field::view_dev_t; + using view_4d = Field::view_dev_t; using const_view_1d = Field::view_dev_t; using const_view_3d = Field::view_dev_t; - static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); - static constexpr int aerosol_categories_ = - mam4::DryDeposition::aerosol_categories; - static constexpr int n_land_type = mam4::DryDeposition::n_land_type; - private: // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -49,15 +50,15 @@ class MAMDryDep final : public scream::AtmosphereProcess { // inputs // FIXME: collect all inputs and outputs together - view_1d fraction_landuse_[mam4::DryDeposition::n_land_type]; - view_1d vlc_trb_[mam4::AeroConfig::num_modes()][aerosol_categories_]; + view_2d fraction_landuse_; + view_3d vlc_trb_; view_2d rho_; - view_2d vlc_dry_[mam4::AeroConfig::num_modes()][aerosol_categories_]; - view_2d vlc_grv_[mam4::AeroConfig::num_modes()][aerosol_categories_]; - view_2d dqdt_tmp_[mam4::aero_model::pcnst]; + view_4d vlc_dry_; + view_4d vlc_grv_; + view_3d dqdt_tmp_; - view_2d qqcw_[mam4::aero_model::pcnst]; + view_3d qqcw_; public: using KT = ekat::KokkosTypes; From 344ce4ea429db569be902812b01e1ec0cb1e24c9 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 18 Jul 2024 09:58:00 -0700 Subject: [PATCH 181/477] fix start date for ARM97 test case --- components/eamxx/cime_config/config_compsets.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index 77c2de92ed32..e8c94534b919 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -99,7 +99,7 @@ 2016-08-01 2020-01-20 1999-07-10 - 1999-06-23 + 1997-06-23 2000-01-01 From 6c384867d50e6d858cf5a19d081c572924edddb8 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 18 Jul 2024 15:46:35 -0600 Subject: [PATCH 182/477] Add ustar and obklen to FM --- .../shoc/eamxx_shoc_process_interface.cpp | 9 ++++++--- .../shoc/eamxx_shoc_process_interface.hpp | 4 +--- .../src/physics/shoc/impl/shoc_main_impl.hpp | 20 +++++++++++-------- .../eamxx/src/physics/shoc/shoc_functions.hpp | 12 +++++++---- .../src/physics/shoc/shoc_functions_f90.cpp | 10 +++++----- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 15d43c82f7c6..80485bf67130 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -88,6 +88,8 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); + add_field("ustar", scalar2d, m/s, grid_name, ps); + add_field("obklen", scalar2d, m, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -157,7 +159,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) #ifdef SCREAM_SMALL_KERNELS , &m_buffer.se_b, &m_buffer.ke_b, &m_buffer.wv_b, &m_buffer.wl_b , &m_buffer.se_a, &m_buffer.ke_a, &m_buffer.wv_a, &m_buffer.wl_a - , &m_buffer.ustar, &m_buffer.kbfs, &m_buffer.obklen, &m_buffer.ustar2, &m_buffer.wstar + , &m_buffer.kbfs, &m_buffer.ustar2, &m_buffer.wstar #endif }; for (int i = 0; i < Buffer::num_1d_scalar_ncol; ++i) { @@ -332,6 +334,8 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) output.pblh = get_field_out("pbl_height").get_view(); output.shoc_ql2 = shoc_ql2; output.tkh = get_field_out("eddy_diff_heat").get_view(); + output.ustar = get_field_out("ustar").get_view(); + output.obklen = get_field_out("obklen").get_view(); // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; @@ -358,9 +362,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) temporaries.ke_a = m_buffer.ke_a; temporaries.wv_a = m_buffer.wv_a; temporaries.wl_a = m_buffer.wl_a; - temporaries.ustar = m_buffer.ustar; temporaries.kbfs = m_buffer.kbfs; - temporaries.obklen = m_buffer.obklen; temporaries.ustar2 = m_buffer.ustar2; temporaries.wstar = m_buffer.wstar; @@ -393,6 +395,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); + add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 6dfadb1eee7b..1695e60dfbc1 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -388,7 +388,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess #ifndef SCREAM_SMALL_KERNELS static constexpr int num_1d_scalar_ncol = 4; #else - static constexpr int num_1d_scalar_ncol = 17; + static constexpr int num_1d_scalar_ncol = 15; #endif static constexpr int num_1d_scalar_nlev = 1; #ifndef SCREAM_SMALL_KERNELS @@ -413,9 +413,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_1d ke_a; uview_1d wv_a; uview_1d wl_a; - uview_1d ustar; uview_1d kbfs; - uview_1d obklen; uview_1d ustar2; uview_1d wstar; #endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 120ed505f261..1e9b4746701f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -118,6 +118,8 @@ void Functions::shoc_main_internal( const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -144,9 +146,9 @@ void Functions::shoc_main_internal( {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); // Local scalars - Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, - se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, - ustar{0}, kbfs{0}, obklen{0}, ustar2{0}, wstar{0}; + Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, + se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, + kbfs{0}, ustar2{0}, wstar{0}; // Scalarize some views for single entry access const auto s_thetal = ekat::scalarize(thetal); @@ -371,6 +373,8 @@ void Functions::shoc_main_internal( const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -397,9 +401,7 @@ void Functions::shoc_main_internal( const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, @@ -625,6 +627,8 @@ Int Functions::shoc_main( const Scalar vw_sfc_s{shoc_input.vw_sfc(i)}; const Scalar phis_s{shoc_input.phis(i)}; Scalar pblh_s{0}; + Scalar ustar_s{0}; + Scalar obklen_s{0}; const auto zt_grid_s = ekat::subview(shoc_input.zt_grid, i); const auto zi_grid_s = ekat::subview(shoc_input.zi_grid, i); @@ -676,7 +680,7 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_s, // Output + pblh_s, ustar_s, obklen_s, shoc_ql2_s, tkh_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables @@ -700,14 +704,14 @@ Int Functions::shoc_main( shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output - shoc_output.pblh, shoc_output.shoc_ql2, shoc_output.tkh, // Output + shoc_output.pblh, shoc_output.ustar, shoc_output.obklen, shoc_output.shoc_ql2, shoc_output.tkh, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables shoc_history_output.wthl_sec, shoc_history_output.wqw_sec, shoc_history_output.wtke_sec, shoc_history_output.uw_sec, shoc_history_output.vw_sec, // Diagnostic Output Variables shoc_history_output.w3, shoc_history_output.wqls_sec, shoc_history_output.brunt, shoc_history_output.isotropy, // Diagnostic Output Variables // Temporaries shoc_temporaries.se_b, shoc_temporaries.ke_b, shoc_temporaries.wv_b, shoc_temporaries.wl_b, shoc_temporaries.se_a, shoc_temporaries.ke_a, shoc_temporaries.wv_a, shoc_temporaries.wl_a, - shoc_temporaries.ustar, shoc_temporaries.kbfs, shoc_temporaries.obklen, shoc_temporaries.ustar2, + shoc_temporaries.kbfs, shoc_temporaries.ustar2, shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi); #endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 74602c688c23..afb9f618d52b 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -159,6 +159,10 @@ struct Functions // planetary boundary layer depth [m] view_1d pblh; + // surface friction velocity [m/s] + view_1d ustar; + // Monin Obukhov length [m] + view_1d obklen; // cloud liquid mixing ratio variance [kg^2/kg^2] view_2d shoc_ql2; // eddy coefficient for heat [m2/s] @@ -211,9 +215,7 @@ struct Functions view_1d ke_a; view_1d wv_a; view_1d wl_a; - view_1d ustar; view_1d kbfs; - view_1d obklen; view_1d ustar2; view_1d wstar; @@ -889,6 +891,8 @@ struct Functions const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -961,6 +965,8 @@ struct Functions const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -987,9 +993,7 @@ struct Functions const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index dc45e2456aa8..b5a2a00d27c8 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2801,7 +2801,9 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, uw_sfc_d (temp_1d_d[index_counter++]), vw_sfc_d (temp_1d_d[index_counter++]), phis_d (temp_1d_d[index_counter++]), - pblh_d ("pblh",shcol); + pblh_d ("pblh",shcol), + ustar_d ("ustar",shcol), + obklen_d ("obklen",shcol); index_counter = 0; view_2d @@ -2878,7 +2880,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, SHF::SHOCInputOutput shoc_input_output{host_dse_d, tke_d, thetal_d, qw_d, horiz_wind_d, wthv_sec_d, qtracers_cxx_d, tk_d, shoc_cldfrac_d, shoc_ql_d}; - SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d, tkh_d}; + SHF::SHOCOutput shoc_output{pblh_d, ustar_d, obklen_d, shoc_ql2_d, tkh_d}; SHF::SHOCHistoryOutput shoc_history_output{shoc_mix_d, w_sec_d, thl_sec_d, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, @@ -2897,9 +2899,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, ke_a ("ke_a", shcol), wv_a ("wv_a", shcol), wl_a ("wl_a", shcol), - ustar ("ustar", shcol), kbfs ("kbfs", shcol), - obklen ("obklen", shcol), ustar2 ("ustar2", shcol), wstar ("wstar", shcol); @@ -2911,7 +2911,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, dz_zi ("dz_zi", shcol, nlevi_packs); SHF::SHOCTemporaries shoc_temporaries{ - se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, + se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, kbfs, ustar2, wstar, rho_zt, shoc_qv, tabs, dz_zt, dz_zi}; #endif From 4a0e4d99cd5ce242c3218605d1104eeb98fe33fa Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 18 Jul 2024 23:29:52 -0700 Subject: [PATCH 183/477] Fixes np1 vs npX tests and clang format --- .../eamxx_mam_dry_deposition_functions.hpp | 97 +++++++++---------- ...x_mam_dry_deposition_process_interface.cpp | 9 +- 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp index 1fdd8fd1243e..f5423e8eb1cf 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -19,8 +19,7 @@ void compute_tendencies( const MAMDryDep::const_view_1d ocnfrac, const MAMDryDep::const_view_1d friction_velocity, const MAMDryDep::const_view_1d aerodynamical_resistance, - MAMDryDep::view_3d qtracers, - MAMDryDep::view_2d fraction_landuse_, + MAMDryDep::view_3d qtracers, MAMDryDep::view_2d fraction_landuse_, const MAMDryDep::const_view_3d dgncur_awet_, const MAMDryDep::const_view_3d wet_dens_, const mam_coupling::DryAtmosphere dry_atm, @@ -34,10 +33,8 @@ void compute_tendencies( MAMDryDep::view_2d aerdepdryis, // work arrays - MAMDryDep::view_2d rho_, - MAMDryDep::view_4d vlc_dry_, - MAMDryDep::view_3d vlc_trb_, - MAMDryDep::view_4d vlc_grv_, + MAMDryDep::view_2d rho_, MAMDryDep::view_4d vlc_dry_, + MAMDryDep::view_3d vlc_trb_, MAMDryDep::view_4d vlc_grv_, MAMDryDep::view_3d dqdt_tmp_) { static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); const auto policy = @@ -112,17 +109,15 @@ void compute_tendencies( dqdt_tmp[i] = ekat::subview(dqdt_tmp_, i, icol); } // Extract qqcw from Prognostics - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](int kk) { - for(int m = 0; m < nmodes; ++m) { - qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = - progs.n_mode_c[m][kk]; - for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) - if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) - qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = - progs.q_aero_c[m][a][kk]; - } - }); // parallel_for nlevs + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < nmodes; ++m) { + qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = + progs.q_aero_c[m][a][kk]; + } + }); // parallel_for nlevs team.team_barrier(); bool ptend_lq[pcnst]; // currently unused mam4::aero_model_drydep( @@ -149,40 +144,38 @@ void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, const mam_coupling::AerosolState dry_aero) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( - 1, nlev); + ncol, nlev); static constexpr int nmodes = mam4::AeroConfig::num_modes(); Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { const int icol = team.league_rank(); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](int kk) { - for(int m = 0; m < nmodes; ++m) { - dry_aero.int_aero_nmr[m](icol, kk) += - ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; - for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) - if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) - dry_aero.int_aero_mmr[m][a](icol, kk) += - ptend_q(icol, kk, - mam4::ConvProc::lmassptrcw_amode(a, m)) * - dt; - } - }); // parallel_for nlevs - }); // parallel_for icol + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < nmodes; ++m) { + dry_aero.int_aero_nmr[m](icol, kk) += + ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + dry_aero.int_aero_mmr[m][a](icol, kk) += + ptend_q(icol, kk, mam4::ConvProc::lmassptrcw_amode(a, m)) * + dt; + } + }); // parallel_for nlevs + }); // parallel_for icol } // Update interstitial aerosols ends // Update cloud borne aerosols using qqcw -void update_cloudborne_mmrs( - const MAMDryDep::view_3d qqcw, const double dt, - const int nlev_, - // output - const mam_coupling::AerosolState dry_aero) { +void update_cloudborne_mmrs(const MAMDryDep::view_3d qqcw, const double dt, + const int nlev_, + // output + const mam_coupling::AerosolState dry_aero) { for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], ekat::subview(qqcw, mam4::ConvProc::numptrcw_amode(m))); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { if(dry_aero.cld_aero_mmr[m][a].data()) { - Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], - ekat::subview(qqcw, mam4::ConvProc::lmassptrcw_amode(a, m))); + Kokkos::deep_copy( + dry_aero.cld_aero_mmr[m][a], + ekat::subview(qqcw, mam4::ConvProc::lmassptrcw_amode(a, m))); } } } @@ -190,19 +183,19 @@ void update_cloudborne_mmrs( // FIXME: remove the following function after implementing file read for landuse void populated_fraction_landuse(MAMDryDep::view_2d flu, const int ncol) { - Kokkos::parallel_for("populated_fraction_landuse", 1, KOKKOS_LAMBDA(int ) { - static constexpr int n_land_type = MAMDryDep::n_land_type; - const Real temp[n_land_type] = - {0.28044346587077795E-003, 0.26634987180780171E-001, - 0.16803558403621365E-001, 0.18076055155371872E-001, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.91803784897907303E+000, 0.17186036997038400E-002, - 0.00000000000000000E+000, 0.00000000000000000E+000, - 0.18448503115578840E-001}; - for(int i = 0; i < n_land_type; ++i) - for(int j = 0; j < ncol; ++j) - flu(i, j) = temp[i]; - }); + Kokkos::parallel_for( + "populated_fraction_landuse", 1, KOKKOS_LAMBDA(int) { + static constexpr int n_land_type = MAMDryDep::n_land_type; + const Real temp[n_land_type] = { + 0.28044346587077795E-003, 0.26634987180780171E-001, + 0.16803558403621365E-001, 0.18076055155371872E-001, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.91803784897907303E+000, 0.17186036997038400E-002, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.18448503115578840E-001}; + for(int i = 0; i < n_land_type; ++i) + for(int j = 0; j < ncol; ++j) flu(i, j) = temp[i]; + }); Kokkos::fence(); } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index b4625387352f..da4e393e749b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -325,9 +325,12 @@ void MAMDryDep::initialize_impl(const RunType run_type) { rho_ = view_2d("rho", ncol_, nlev_); ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); - vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_, nlev_); - vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_, nlev_); - vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_); + vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_); qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); dqdt_tmp_ = view_3d("dqdt_tmp_", pcnst, ncol_, nlev_); From 047802bf4eec3cdd341c9ac7bc1f7b6869ce8950 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 06:29:57 -0700 Subject: [PATCH 184/477] Updates mam4xx submodule-pointed to main for wetscav qqcw update --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index cafdbc1b8813..316817f6925d 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit cafdbc1b88134038aff12c1cb00110c6397a4544 +Subproject commit 316817f6925d5d0380f624402c0c5ede3070e334 From 77fcad25dd09d878265857b5f5d69fb139c2e17c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 07:02:19 -0700 Subject: [PATCH 185/477] point mam4xx submodule to main to get Thread Vector changes among others --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index a8aa56ba915c..316817f6925d 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e +Subproject commit 316817f6925d5d0380f624402c0c5ede3070e334 From 63175dfbd18a2453f679752b8b4ccd80c60873de Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 19 Jul 2024 15:04:44 -0400 Subject: [PATCH 186/477] fix capture this warning in mam micro --- .../physics/mam/eamxx_mam_microphysics_process_interface.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7ea2adfddb63..8b93936148ce 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,6 +317,8 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index @@ -448,7 +450,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, some_step, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, From 98c8ca1a7be2e864992f8ef894f3dfc46d0ee163 Mon Sep 17 00:00:00 2001 From: Hassan Beydoun Date: Fri, 19 Jul 2024 13:05:12 -0700 Subject: [PATCH 187/477] add output variable for rain effective diameter --- components/eam/src/physics/p3/scream/micro_p3_interface.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 index edea27959d25..11d13c71a7a7 100644 --- a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 @@ -834,6 +834,8 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) real(rtype) :: icwmrst(pcols,pver) ! stratus water mixing ratio - on grid real(rtype) :: rho(pcols,pver) real(rtype) :: reff_rain(pcols,pver) + real(rtype) :: deff_rain(pcols,pver) + real(rtype) :: col_location(pcols,3),tmp_loc(pcols) ! Array of column lon (index 1) and lat (index 2) integer :: tmpi_loc(pcols) ! Global column index temp array @@ -1358,6 +1360,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) aqrain = 0._rtype anrain = 0._rtype freqr = 0._rtype + deff_rain = reff_rain*2._rtype*1.e6_rtype !rain effective diameter in microns ! Prognostic precipitation where (rain(:ncol,top_lev:) >= 1.e-7_rtype) aqrain(:ncol,top_lev:) = rain(:ncol,top_lev:) * cld_frac_r(:ncol,top_lev:) @@ -1412,6 +1415,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) call outfld('FREQI', freqi, pcols, lchnk) call outfld('FREQR', freqr, pcols, lchnk) call outfld('CDNUMC', cdnumc, pcols, lchnk) + call outfld('ADRAIN', deff_rain, pcols, lchnk) call outfld('CLOUDFRAC_LIQ_MICRO', cld_frac_l, pcols, lchnk) call outfld('CLOUDFRAC_ICE_MICRO', cld_frac_i, pcols, lchnk) From d92ea27914b7b87571b6d53355d12dac0e42606a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 13:56:26 -0700 Subject: [PATCH 188/477] Declared local variables for device (recent fixes from Naser) --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 316817f6925d..5495c27a6df7 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 316817f6925d5d0380f624402c0c5ede3070e334 +Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d From c75615f181d95850610e80c7f839f6ee4150da29 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 14:11:49 -0700 Subject: [PATCH 189/477] Submodule update: declare local vrs for device code --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 316817f6925d..5495c27a6df7 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 316817f6925d5d0380f624402c0c5ede3070e334 +Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d From b86b4df708fc6cb91d760177162c6183dfd0a0d2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 14:27:06 -0700 Subject: [PATCH 190/477] Adds 2d fields from land model and updates mam4xx sub to point to main --- .../atmosphere_surface_coupling_importer.cpp | 5 +++++ .../src/mct_coupling/scream_cpl_indices.F90 | 10 +++++++-- ...x_mam_dry_deposition_process_interface.cpp | 22 +++++++++---------- .../single-process/mam/drydep/input.yaml | 10 ++++----- externals/mam4xx | 2 +- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 52deba16e8d3..7573993a3189 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -52,6 +52,11 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("snow_depth_land", scalar2d_layout, m, grid_name); add_field("ocnfrac", scalar2d_layout, nondim, grid_name); add_field("landfrac", scalar2d_layout, nondim, grid_name); + add_field("icefrac", scalar2d_layout, nondim, grid_name); + // Friction velocity [m/s] + add_field("fv", scalar2d_layout, m/s, grid_name); + // Aerodynamical resistance + add_field("ram1", scalar2d_layout, s/m, grid_name); } // ========================================================================================= void SurfaceCouplingImporter::setup_surface_coupling_data(const SCDataManager &sc_data_manager) diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index df96edab0f06..e754d2fef1b0 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -6,7 +6,7 @@ module scream_cpl_indices private ! Focus only on the ones that scream imports/exports (subsets of x2a and a2x) - integer, parameter, public :: num_scream_imports = 16 + integer, parameter, public :: num_scream_imports = 19 integer, parameter, public :: num_scream_exports = 17 integer, public :: num_cpl_imports, num_cpl_exports, import_field_size, export_field_size @@ -89,6 +89,9 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_field_names(14) = 'surf_evap' import_field_names(15) = 'ocnfrac' import_field_names(16) = 'landfrac' + import_field_names(17) = 'icefrac' + import_field_names(18) = 'fv' + import_field_names(19) = 'ram1' ! CPL indices import_cpl_indices(1) = mct_avect_indexra(x2a,'Sx_avsdr') @@ -106,7 +109,10 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(13) = mct_avect_indexra(x2a,'Faxx_sen') import_cpl_indices(14) = mct_avect_indexra(x2a,'Faxx_evap') import_cpl_indices(15) = mct_avect_indexra(x2a,'Sf_ofrac') - import_cpl_indices(16) = mct_avect_indexra(x2a,'Sf_lfrac') + import_cpl_indices(16) = mct_avect_indexra(x2a,'Sf_lfrac') + import_cpl_indices(17) = mct_avect_indexra(x2a,'Sf_ifrac') + import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_fv') + import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_ram1') ! Vector components import_vector_components(11) = 0 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index da4e393e749b..70f69761d18b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -126,22 +126,22 @@ void MAMDryDep::set_grids( add_field("surface_friction_velocty", scalar2d, m / s, grid_name); // Land fraction [fraction] - add_field("land_fraction", scalar2d, nondim, grid_name); + add_field("landfrac", scalar2d, nondim, grid_name); // Friction velocity from land model [m/s] - add_field("friction_velocity", scalar2d, m / s, grid_name); + add_field("fv", scalar2d, m / s, grid_name); // Aerodynamical resistance from land model [s/m] - add_field("aerodynamical_resistance", scalar2d, s / m, grid_name); + add_field("ram1", scalar2d, s / m, grid_name); //----------- Variables from coupler (ice component)--------- // Ice fraction [unitless] - add_field("ice_fraction", scalar2d, nondim, grid_name); + add_field("icefrac", scalar2d, nondim, grid_name); //----------- Variables from coupler (ocean component)--------- // Ocean fraction [unitless] - add_field("ocean_fraction", scalar2d, nondim, grid_name); + add_field("ocnfrac", scalar2d, nondim, grid_name); //----------- Variables from other mam4xx processes ------------ // Geometric mean wet diameter for number distribution [m] @@ -369,18 +369,16 @@ void MAMDryDep::run_impl(const double dt) { auto obukhov_length_ = get_field_in("Obukhov_length").get_view(); // Land fraction [unitless] - auto land_fraction_ = get_field_in("land_fraction").get_view(); + auto land_fraction_ = get_field_in("landfrac").get_view(); // Ice fraction [unitless] - auto ice_fraction_ = get_field_in("ice_fraction").get_view(); + auto ice_fraction_ = get_field_in("icefrac").get_view(); // Ocean fraction [unitless] - auto ocean_fraction_ = - get_field_in("ocean_fraction").get_view(); + auto ocean_fraction_ = get_field_in("ocnfrac").get_view(); // Friction velocity from land model [m/s] - auto friction_velocity_ = - get_field_in("friction_velocity").get_view(); + auto friction_velocity_ = get_field_in("fv").get_view(); // Aerodynamical resistance from land model [s/m] auto aerodynamical_resistance_ = - get_field_in("aerodynamical_resistance").get_view(); + get_field_in("ram1").get_view(); // Sfc friction velocity or ustar [m/s] auto surface_friction_velocty_ = get_field_in("surface_friction_velocty").get_view(); diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 1de57ad7e682..685a68fcea04 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -29,12 +29,12 @@ initial_conditions: w_updraft: 1e-5 pbl_height: 0.0 Obukhov_length: -0.48167567011091336E+002 - friction_velocity: 0.47569674433601039E+000 - land_fraction: 0.14675684171817760E+000 - ice_fraction: 0.00000000000000000E+000 - ocean_fraction: 0.85324315828182240E+000 + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 surface_friction_velocty: 0.27482164092221828E+000 - aerodynamical_resistance: 0.45506166067091662E+002 + ram1: 0.45506166067091662E+002 dgnumwet: 0.41417721820867320E-007 wetdens: 0.15100083211582764E+004 diff --git a/externals/mam4xx b/externals/mam4xx index 5e7ff7200b49..5495c27a6df7 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 5e7ff7200b4910a43fbd254951c6ff61b6463e30 +Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d From ed2e0ea624c4e479cef454b9e5e40e0cff1148d6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 15:02:51 -0700 Subject: [PATCH 191/477] Adds shoc+drydep unit test --- .../multi-process/physics_only/CMakeLists.txt | 1 + .../mam/shoc_mam4_drydep/CMakeLists.txt | 41 +++++++++++ .../mam/shoc_mam4_drydep/input.yaml | 69 +++++++++++++++++++ .../mam/shoc_mam4_drydep/output.yaml | 69 +++++++++++++++++++ 4 files changed, 180 insertions(+) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index b9c13234796b..716152324e57 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -5,6 +5,7 @@ if (SCREAM_DOUBLE_PRECISION) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) add_subdirectory(mam/shoc_mam4_aci) + add_subdirectory(mam/shoc_mam4_drydep) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt new file mode 100644 index 000000000000..9f6fb8733c2c --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_mam4_drydep) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc mam + LABELS shoc physics mam4_drydep + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc physics mam4_drydep + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml new file mode 100644 index 000000000000..035c90c3a6a6 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml @@ -0,0 +1,69 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic,mam4_drydep] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables needed for SHOC + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variables needed for mam4-drydep + fraction_landuse: 0.2 + w_updraft: 1e-5 + pbl_height: 0.0 + Obukhov_length: -0.48167567011091336E+002 + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 + surface_friction_velocty: 0.27482164092221828E+000 + ram1: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml new file mode 100644 index 000000000000..d8c1291bc3c9 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml @@ -0,0 +1,69 @@ +%YAML 1.1 +--- +filename_prefix: shoc_mam4_drydep_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 9b937f91e22bbcb3abce0cbad927001913ac4299 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 18 Jul 2024 15:46:35 -0600 Subject: [PATCH 192/477] Add ustar and obklen to FM --- .../shoc/eamxx_shoc_process_interface.cpp | 9 ++++++--- .../shoc/eamxx_shoc_process_interface.hpp | 4 +--- .../src/physics/shoc/impl/shoc_main_impl.hpp | 20 +++++++++++-------- .../eamxx/src/physics/shoc/shoc_functions.hpp | 12 +++++++---- .../src/physics/shoc/shoc_functions_f90.cpp | 10 +++++----- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 15d43c82f7c6..80485bf67130 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -88,6 +88,8 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); + add_field("ustar", scalar2d, m/s, grid_name, ps); + add_field("obklen", scalar2d, m, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -157,7 +159,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) #ifdef SCREAM_SMALL_KERNELS , &m_buffer.se_b, &m_buffer.ke_b, &m_buffer.wv_b, &m_buffer.wl_b , &m_buffer.se_a, &m_buffer.ke_a, &m_buffer.wv_a, &m_buffer.wl_a - , &m_buffer.ustar, &m_buffer.kbfs, &m_buffer.obklen, &m_buffer.ustar2, &m_buffer.wstar + , &m_buffer.kbfs, &m_buffer.ustar2, &m_buffer.wstar #endif }; for (int i = 0; i < Buffer::num_1d_scalar_ncol; ++i) { @@ -332,6 +334,8 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) output.pblh = get_field_out("pbl_height").get_view(); output.shoc_ql2 = shoc_ql2; output.tkh = get_field_out("eddy_diff_heat").get_view(); + output.ustar = get_field_out("ustar").get_view(); + output.obklen = get_field_out("obklen").get_view(); // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; @@ -358,9 +362,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) temporaries.ke_a = m_buffer.ke_a; temporaries.wv_a = m_buffer.wv_a; temporaries.wl_a = m_buffer.wl_a; - temporaries.ustar = m_buffer.ustar; temporaries.kbfs = m_buffer.kbfs; - temporaries.obklen = m_buffer.obklen; temporaries.ustar2 = m_buffer.ustar2; temporaries.wstar = m_buffer.wstar; @@ -393,6 +395,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); + add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 6dfadb1eee7b..1695e60dfbc1 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -388,7 +388,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess #ifndef SCREAM_SMALL_KERNELS static constexpr int num_1d_scalar_ncol = 4; #else - static constexpr int num_1d_scalar_ncol = 17; + static constexpr int num_1d_scalar_ncol = 15; #endif static constexpr int num_1d_scalar_nlev = 1; #ifndef SCREAM_SMALL_KERNELS @@ -413,9 +413,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_1d ke_a; uview_1d wv_a; uview_1d wl_a; - uview_1d ustar; uview_1d kbfs; - uview_1d obklen; uview_1d ustar2; uview_1d wstar; #endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 120ed505f261..1e9b4746701f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -118,6 +118,8 @@ void Functions::shoc_main_internal( const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -144,9 +146,9 @@ void Functions::shoc_main_internal( {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); // Local scalars - Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, - se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, - ustar{0}, kbfs{0}, obklen{0}, ustar2{0}, wstar{0}; + Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, + se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, + kbfs{0}, ustar2{0}, wstar{0}; // Scalarize some views for single entry access const auto s_thetal = ekat::scalarize(thetal); @@ -371,6 +373,8 @@ void Functions::shoc_main_internal( const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -397,9 +401,7 @@ void Functions::shoc_main_internal( const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, @@ -625,6 +627,8 @@ Int Functions::shoc_main( const Scalar vw_sfc_s{shoc_input.vw_sfc(i)}; const Scalar phis_s{shoc_input.phis(i)}; Scalar pblh_s{0}; + Scalar ustar_s{0}; + Scalar obklen_s{0}; const auto zt_grid_s = ekat::subview(shoc_input.zt_grid, i); const auto zi_grid_s = ekat::subview(shoc_input.zi_grid, i); @@ -676,7 +680,7 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_s, // Output + pblh_s, ustar_s, obklen_s, shoc_ql2_s, tkh_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables @@ -700,14 +704,14 @@ Int Functions::shoc_main( shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output - shoc_output.pblh, shoc_output.shoc_ql2, shoc_output.tkh, // Output + shoc_output.pblh, shoc_output.ustar, shoc_output.obklen, shoc_output.shoc_ql2, shoc_output.tkh, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables shoc_history_output.wthl_sec, shoc_history_output.wqw_sec, shoc_history_output.wtke_sec, shoc_history_output.uw_sec, shoc_history_output.vw_sec, // Diagnostic Output Variables shoc_history_output.w3, shoc_history_output.wqls_sec, shoc_history_output.brunt, shoc_history_output.isotropy, // Diagnostic Output Variables // Temporaries shoc_temporaries.se_b, shoc_temporaries.ke_b, shoc_temporaries.wv_b, shoc_temporaries.wl_b, shoc_temporaries.se_a, shoc_temporaries.ke_a, shoc_temporaries.wv_a, shoc_temporaries.wl_a, - shoc_temporaries.ustar, shoc_temporaries.kbfs, shoc_temporaries.obklen, shoc_temporaries.ustar2, + shoc_temporaries.kbfs, shoc_temporaries.ustar2, shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi); #endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 74602c688c23..afb9f618d52b 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -159,6 +159,10 @@ struct Functions // planetary boundary layer depth [m] view_1d pblh; + // surface friction velocity [m/s] + view_1d ustar; + // Monin Obukhov length [m] + view_1d obklen; // cloud liquid mixing ratio variance [kg^2/kg^2] view_2d shoc_ql2; // eddy coefficient for heat [m2/s] @@ -211,9 +215,7 @@ struct Functions view_1d ke_a; view_1d wv_a; view_1d wl_a; - view_1d ustar; view_1d kbfs; - view_1d obklen; view_1d ustar2; view_1d wstar; @@ -889,6 +891,8 @@ struct Functions const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -961,6 +965,8 @@ struct Functions const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -987,9 +993,7 @@ struct Functions const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index dc45e2456aa8..b5a2a00d27c8 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2801,7 +2801,9 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, uw_sfc_d (temp_1d_d[index_counter++]), vw_sfc_d (temp_1d_d[index_counter++]), phis_d (temp_1d_d[index_counter++]), - pblh_d ("pblh",shcol); + pblh_d ("pblh",shcol), + ustar_d ("ustar",shcol), + obklen_d ("obklen",shcol); index_counter = 0; view_2d @@ -2878,7 +2880,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, SHF::SHOCInputOutput shoc_input_output{host_dse_d, tke_d, thetal_d, qw_d, horiz_wind_d, wthv_sec_d, qtracers_cxx_d, tk_d, shoc_cldfrac_d, shoc_ql_d}; - SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d, tkh_d}; + SHF::SHOCOutput shoc_output{pblh_d, ustar_d, obklen_d, shoc_ql2_d, tkh_d}; SHF::SHOCHistoryOutput shoc_history_output{shoc_mix_d, w_sec_d, thl_sec_d, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, @@ -2897,9 +2899,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, ke_a ("ke_a", shcol), wv_a ("wv_a", shcol), wl_a ("wl_a", shcol), - ustar ("ustar", shcol), kbfs ("kbfs", shcol), - obklen ("obklen", shcol), ustar2 ("ustar2", shcol), wstar ("wstar", shcol); @@ -2911,7 +2911,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, dz_zi ("dz_zi", shcol, nlevi_packs); SHF::SHOCTemporaries shoc_temporaries{ - se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, + se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, kbfs, ustar2, wstar, rho_zt, shoc_qv, tabs, dz_zt, dz_zi}; #endif From 155ec3c142e54ac65ce080e953715b8f5d6ece36 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:10:21 -0500 Subject: [PATCH 193/477] Add ums and ums002 to load libunwind 1.6.2 After recent Frontier maintenance, ums/default and ums002/default are required to load libunwind/1.6.2. --- cime_config/machines/config_machines.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index c04dff5976cd..3617bb88f9ec 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1437,6 +1437,8 @@ PrgEnv-cray craype-accel-amd-gfx90a rocm/5.4.0 + ums/default + ums002/default libunwind/1.6.2 From 555aa19b0967bb2dd6a570f8bf74aadaa6eb8f67 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:12:27 -0500 Subject: [PATCH 194/477] Add to scream_scorpio_types.hpp to use std::int64_t This fixes a known build error after recent Frontier maintenance: no type named 'int64_t' in namespace 'std'; did you mean simply 'int64_t'? --- components/eamxx/src/share/io/scream_scorpio_types.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/share/io/scream_scorpio_types.hpp b/components/eamxx/src/share/io/scream_scorpio_types.hpp index 645b1b26937b..3b16cebc06c8 100644 --- a/components/eamxx/src/share/io/scream_scorpio_types.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_types.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace scream { namespace scorpio { From 805ab1f68d5dd2599a1eaf90e16c26fc8cf47d08 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:16:28 -0500 Subject: [PATCH 195/477] Explicitly add gcc/12.2.0 lib path to CMAKE_EXE_LINKER_FLAGS This is a simple workaround to fix scream linking errors after recent Frontier maintenance before we find better solutions. --- .../cmake_macros/crayclang-scream_frontier-scream-gpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index afcca8f479e5..cc90b369244a 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -19,7 +19,7 @@ endif() string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64 -L/opt/gcc/12.2.0/snos/lib64") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{ROCM_PATH}/include") # Crusher: this resolves a crash in mct in docn init From 2f4352dd38a5b93fd9dd17c8d814d6ca7cb1e99d Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 19 Jul 2024 15:48:20 -0700 Subject: [PATCH 196/477] filter aodvis by day indices --- components/eamxx/src/diagnostics/aodvis.cpp | 19 ++++++-- .../src/diagnostics/tests/aodvis_test.cpp | 46 ++++++++++++++++--- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index efb9b5ba9510..ac16c0033730 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -2,6 +2,8 @@ #include +#include "share/util/scream_universal_constants.hpp" + namespace scream { AODVis::AODVis(const ekat::Comm &comm, const ekat::ParameterList ¶ms) @@ -23,11 +25,13 @@ void AODVis::set_grids( m_nlevs = grid->get_num_vertical_levels(); // Define layouts we need (both inputs and outputs) - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,m_swbands,"swband"); - FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, m_swbands, "swband"); + FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); // The fields required for this diagnostic to be computed add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + add_field("sunlit", scalar1d_layout, nondim, grid_name); // Construct and allocate the aodvis field FieldIdentifier fid("AerosolOpticalDepth550nm", scalar1d_layout, nondim, @@ -41,18 +45,25 @@ void AODVis::compute_diagnostic_impl() { using MT = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; + Real var_fill_value = constants::DefaultFillValue().value; + const auto aod = m_diagnostic_output.get_view(); const auto tau_vis = get_field_in("aero_tau_sw") .subfield(1, m_vis_bnd) .get_view(); + const auto sunlit = get_field_in("sunlit").get_view(); const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); - auto tau_icol = ekat::subview(tau_vis, icol); - aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + if(sunlit(icol) == 0.0) { + aod(icol) = var_fill_value; + } else { + auto tau_icol = ekat::subview(tau_vis, icol); + aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + } }); } diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 140114812215..15de175da67d 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -1,10 +1,11 @@ +#include + #include "catch2/catch.hpp" #include "diagnostics/register_diagnostics.hpp" #include "share/field/field_utils.hpp" #include "share/grid/mesh_free_grids_manager.hpp" #include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_utils.hpp" - +#include "share/util/scream_universal_constants.hpp" namespace scream { std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, @@ -30,6 +31,10 @@ TEST_CASE("aodvis") { using namespace ShortFieldTagsNames; using namespace ekat::units; + Real var_fill_value = constants::DefaultFillValue().value; + + Real some_limit = 0.0025; + // A world comm ekat::Comm comm(MPI_COMM_WORLD); @@ -40,7 +45,7 @@ TEST_CASE("aodvis") { // Create a grids manager - single column for these tests constexpr int nlevs = 33; - const int ngcols = 2 * comm.size(); + const int ngcols = 10 * comm.size(); int nbnds = eamxx_swbands(); int swvis = eamxx_vis_swband_idx(); @@ -49,16 +54,23 @@ TEST_CASE("aodvis") { auto grid = gm->get_grid("Physics"); // Input (randomized) tau - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,nbnds,"swband"); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, nbnds, "swband"); FieldIdentifier tau_fid("aero_tau_sw", scalar3d_swband_layout, nondim, grid->name()); Field tau(tau_fid); tau.allocate_view(); tau.get_header().get_tracking().update_time_stamp(t0); + // Input (randomized) sunlit + FieldLayout scalar2d_layout = grid->get_2d_scalar_layout(); + FieldIdentifier sunlit_fid("sunlit", scalar2d_layout, nondim, grid->name()); + Field sunlit(sunlit_fid); + sunlit.allocate_view(); + sunlit.get_header().get_tracking().update_time_stamp(t0); // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(0, 0.05); + RPDF pdf(0, 0.005); auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -71,18 +83,33 @@ TEST_CASE("aodvis") { // Randomize tau randomize(tau, engine, pdf); + // Randomize sunlit + randomize(sunlit, engine, pdf); + // Create and set up the diagnostic ekat::ParameterList params; auto diag = diag_factory.create("AerosolOpticalDepth550nm", comm, params); diag->set_grids(gm); diag->set_required_field(tau); + diag->set_required_field(sunlit); diag->initialize(t0, RunType::Initial); + auto sun_h = sunlit.get_view(); + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + // zero out all sun_h if below 0.05 + if(sun_h(icol) < some_limit) { + sun_h(icol) = 0.0; + } + } + // sync to device + sunlit.sync_to_dev(); + // Run diag diag->compute_diagnostic(); // Check result tau.sync_to_host(); + sunlit.sync_to_host(); diag->get_diagnostic().sync_to_host(); const auto tau_h = tau.get_view(); @@ -93,12 +120,17 @@ TEST_CASE("aodvis") { auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - for(int ilev = 0; ilev < nlevs; ++ilev) { - aod_t(icol) += tau_h(icol, swvis, ilev); + if(sun_h(icol) < some_limit) { + aod_t(icol) = var_fill_value; + } else { + for(int ilev = 0; ilev < nlevs; ++ilev) { + aod_t(icol) += tau_h(icol, swvis, ilev); + } } } aod_hf.sync_to_dev(); aod_tf.sync_to_dev(); + // Workaround for non-bfb behavior of view_reduction() in release builds if(SCREAM_BFB_TESTING) { REQUIRE(views_are_equal(aod_hf, aod_tf)); From 3baa1b1827c72dc8c36b006b49017a8136d4a70b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Jul 2024 16:01:44 -0700 Subject: [PATCH 197/477] Uses shoc output variables for obklen and ustar in drydep --- .../mam/eamxx_mam_dry_deposition_process_interface.cpp | 9 ++++----- .../physics_only/mam/shoc_mam4_drydep/input.yaml | 2 -- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 70f69761d18b..047878796d9a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -120,10 +120,10 @@ void MAMDryDep::set_grids( //----------- Variables from coupler (land component)--------- // Obukhov length [m] - add_field("Obukhov_length", scalar2d, m, grid_name); + add_field("obklen", scalar2d, m, grid_name); // Surface friction velocty or ustar[m/s] - add_field("surface_friction_velocty", scalar2d, m / s, grid_name); + add_field("ustar", scalar2d, m / s, grid_name); // Land fraction [fraction] add_field("landfrac", scalar2d, nondim, grid_name); @@ -366,8 +366,7 @@ void MAMDryDep::run_impl(const double dt) { // Wet density of interstitial aerosol [kg/m3] auto wet_dens_ = get_field_in("wetdens").get_view(); // Obukhov length [m] - auto obukhov_length_ = - get_field_in("Obukhov_length").get_view(); + auto obukhov_length_ = get_field_in("obklen").get_view(); // Land fraction [unitless] auto land_fraction_ = get_field_in("landfrac").get_view(); // Ice fraction [unitless] @@ -381,7 +380,7 @@ void MAMDryDep::run_impl(const double dt) { get_field_in("ram1").get_view(); // Sfc friction velocity or ustar [m/s] auto surface_friction_velocty_ = - get_field_in("surface_friction_velocty").get_view(); + get_field_in("ustar").get_view(); // ------------------------------------------------------------- // Output fields for the process diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml index 035c90c3a6a6..c451008adc76 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml @@ -53,12 +53,10 @@ initial_conditions: fraction_landuse: 0.2 w_updraft: 1e-5 pbl_height: 0.0 - Obukhov_length: -0.48167567011091336E+002 fv: 0.47569674433601039E+000 landfrac: 0.14675684171817760E+000 icefrac: 0.00000000000000000E+000 ocnfrac: 0.85324315828182240E+000 - surface_friction_velocty: 0.27482164092221828E+000 ram1: 0.45506166067091662E+002 dgnumwet: 0.41417721820867320E-007 wetdens: 0.15100083211582764E+004 From cc0c4ce0d2f7171fdbc229780194e8905af66795 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Jul 2024 07:27:07 -0700 Subject: [PATCH 198/477] Adds missing links for obklen and ustar in shoc --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp | 1 - components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 80485bf67130..657f9487611c 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -395,7 +395,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); - add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 1e9b4746701f..38758bf04a12 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -685,7 +685,9 @@ Int Functions::shoc_main( wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables - shoc_output.pblh(i) = pblh_s; + shoc_output.obklen(i) = obklen_s; + shoc_output.ustar(i) = ustar_s; + shoc_output.pblh(i) s= pblh_s; }); Kokkos::fence(); #else From d71781f58999349579e6a88d7378ffc05a5741d2 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 08:43:37 -0600 Subject: [PATCH 199/477] Remove obklen postcondition check --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 80485bf67130..657f9487611c 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -395,7 +395,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); - add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. From 6e5097d6a8b2922d44b845fdff5dff0eeebbc78b Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 08:44:17 -0600 Subject: [PATCH 200/477] Set output correctly for new variables --- components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 1e9b4746701f..11383e772454 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -686,6 +686,8 @@ Int Functions::shoc_main( w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables shoc_output.pblh(i) = pblh_s; + shoc_output.ustar(i) = ustar_s; + shoc_output.obklen(i) = obklen_s; }); Kokkos::fence(); #else From f25ec79b6b7c0c46de11d4b9795fcdaf9550a94a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Jul 2024 07:46:10 -0700 Subject: [PATCH 201/477] Fixes a typo --- components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 38758bf04a12..807eccdfe559 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -687,7 +687,7 @@ Int Functions::shoc_main( shoc_output.obklen(i) = obklen_s; shoc_output.ustar(i) = ustar_s; - shoc_output.pblh(i) s= pblh_s; + shoc_output.pblh(i) = pblh_s; }); Kokkos::fence(); #else From 45af3b53e5aee4d9a54a79e78f3066ec5ecf1fcc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Jul 2024 11:56:15 -0700 Subject: [PATCH 202/477] Adds a multi-process test with dynamics --- .../dynamics_physics/CMakeLists.txt | 1 + .../CMakeLists.txt | 109 +++++++++++++++ .../input.yaml | 109 +++++++++++++++ .../output.yaml | 124 ++++++++++++++++++ 4 files changed, 343 insertions(+) create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml create mode 100644 components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml diff --git a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 64ac2187c280..51d0bd456479 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -13,6 +13,7 @@ if (SCREAM_DOUBLE_PRECISION) # initial conditions. #add_subdirectory(homme_mam4xx_pg2) add_subdirectory(mam/homme_shoc_cld_p3_mam_optics_rrtmgp) + add_subdirectory(mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep) endif() endif() endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt new file mode 100644 index 000000000000..e3c562cfcd13 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt @@ -0,0 +1,109 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 41) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction ${dynLibName} shoc p3 scream_rrtmgp mam + LABELS dynamics shoc cld p3 rrtmgp physics mam4_optics mam4_aci mam4_drydep + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 2) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 1) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp mam4_optics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml new file mode 100644 index 000000000000..1857bc534dd2 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml @@ -0,0 +1,109 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + mass_column_conservation_error_tolerance: 1e-3 + energy_column_conservation_error_tolerance: 1e-4 + column_conservation_checks_fail_handling_type: Warning + property_check_data_fields: [phis] + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + phis : 1.0 + + #variables required for shoc + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variables required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + + #variables required for mam4_aci + dgnum: 1e-3 + + #mam4 drydep + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 + ram1: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mac_mic,mam4_optics,rrtmgp,mam4_drydep] + schedule_type: Sequential + Type: Group + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + schedule_type: Sequential + Type: Group + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 + p3: + do_prescribed_ccn: false + enable_column_conservation_checks: true + max_total_ni: 740.0e3 + shoc: + enable_column_conservation_checks: true + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + enable_column_conservation_checks: true + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml new file mode 100644 index 000000000000..f2408ab189a5 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml @@ -0,0 +1,124 @@ +%YAML 1.1 +--- +filename_prefix: homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + # SHOC + - cldfrac_liq + - eddy_diff_mom + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + HOMME + - horiz_winds + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + HOMME + - T_mid + #mam_optics + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + # Diagnostics + - T_mid_at_lev_2 + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_500mb + - T_mid_at_500hPa + - T_mid_at_50000Pa + - PotentialTemperature + - AtmosphereDensity + - Exner + - VirtualTemperature + - z_int + - geopotential_int_at_lev_2 + - z_mid_at_500mb + - geopotential_mid + - dz + - DryStaticEnergy + - SeaLevelPressure + - LiqWaterPath + - IceWaterPath + - VapWaterPath + - RainWaterPath + - RimeWaterPath + - ShortwaveCloudForcing + - LongwaveCloudForcing + - RelativeHumidity + - ZonalVapFlux + - MeridionalVapFlux + - PotentialTemperature_at_model_top + - PotentialTemperature_at_500mb + #drydep + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols + # GLL output for homme states. These + # represent all current possible homme + # states available. + Dynamics: + Field Names: + - v_dyn + - vtheta_dp_dyn + - dp3d_dyn + - phi_int_dyn + - ps_dyn + - phis_dyn + - omega_dyn + - Qdp_dyn + IO Grid Name: Physics GLL + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: true +... From f130c27bf6375d16a470e43af9cbebbb5c6af0dc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Jul 2024 13:14:23 -0700 Subject: [PATCH 203/477] Adds variables for drydep only CIME simulation --- components/eamxx/cime_config/namelist_defaults_scream.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index e586c6ce2151..e3da0b0c9eee 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -493,8 +493,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0,0.0 - 2.6e-08 - 0.0 + 2.6e-08 + 0.41417721820867320E-007 + 0.15100083211582764E+004 + 0.0 0.0 0.0 0.0 From 076bf9195e3105453150a5844876543dfcfa1310 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 18:38:38 -0700 Subject: [PATCH 204/477] Fixes dimensions of dry dep flux variables and fixes input.yaml for a test --- .../eamxx_mam_dry_deposition_process_interface.cpp | 11 ++++++++--- .../eamxx/tests/single-process/mam/drydep/input.yaml | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 047878796d9a..433c9d4f6063 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -48,6 +48,11 @@ void MAMDryDep::set_grids( const FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; const FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // layout for 2D (ncol, pcnst) + constexpr int pcnst = mam4::aero_model::pcnst; + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constants"); + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points auto make_layout = [](const std::vector &extents, @@ -198,11 +203,11 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------- // FIXME: These are diagnostics, remove them from FM after initial evaluation // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_cloud_borne_aerosols", scalar3d_mid, + add_field("deposition_flux_of_cloud_borne_aerosols", scalar2d_pcnct, 1 / m2 / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_interstitial_aerosols", scalar3d_mid, - 1 / m2 / s, grid_name); + add_field("deposition_flux_of_interstitial_aerosols", + scalar2d_pcnct, 1 / m2 / s, grid_name); } // set_grids // ================================================================ diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml index 685a68fcea04..86c6dd8766a2 100644 --- a/components/eamxx/tests/single-process/mam/drydep/input.yaml +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -28,12 +28,12 @@ initial_conditions: fraction_landuse: 0.2 w_updraft: 1e-5 pbl_height: 0.0 - Obukhov_length: -0.48167567011091336E+002 + obklen: -0.48167567011091336E+002 fv: 0.47569674433601039E+000 landfrac: 0.14675684171817760E+000 icefrac: 0.00000000000000000E+000 ocnfrac: 0.85324315828182240E+000 - surface_friction_velocty: 0.27482164092221828E+000 + ustar: 0.27482164092221828E+000 ram1: 0.45506166067091662E+002 dgnumwet: 0.41417721820867320E-007 wetdens: 0.15100083211582764E+004 From dbaeea26beec89d0a7a72187fc813766cabd2b5d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 25 Jul 2024 08:40:52 -0600 Subject: [PATCH 205/477] EAMxx: add runtime and buildnml-time checks on output frequency Ensures dt_output is not smaller than dt_atm --- components/eamxx/cime_config/eamxx_buildnml.py | 17 +++++++++++++++++ .../src/share/io/scream_output_manager.cpp | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 0c7ed8e52aac..121506f7eff3 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -1017,6 +1017,23 @@ def do_cime_vars_on_yaml_output_files(case, caseroot): print (" - setting skip_t0_output=true\n") print (" - setting freq and freq_units to HIST_N and HIST_OPTION respectively\n") + # If frequency_units is not nsteps, verify that we don't request + # a frequency faster than the model timestep + if content['output_control']['frequency_units'] in ['nsecs','nmins','nhours']: + freq = content['output_control']['Frequency'] + units = content['output_control']['frequency_units'] + dt_out = 1 if units=="nsecs" else 60 if units=="nmins" else 3600 + dt_out = dt_out*int(freq) + + dt_atm = 86400 / case.get_value("ATM_NCPL") + expect (dt_atm<=dt_out, + "Cannot have output frequency faster than atm timestep.\n" + f" yaml file: {fn.strip()}\n" + f" Frequency: {freq}\n" + f" frequency_units: {units}\n" + f" ATM_NCPL: {case.get_value('ATM_NCPL')}\n" + f" This yields dt_atm={dt_atm} > dt_output={dt_out}. Please, adjust 'Frequency' and/or 'frequency_units'\n") + ordered_dump(content, open(dst_yaml, "w")) output_yaml_files.append(dst_yaml) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 35b1c753b03e..539204c66a1b 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -328,6 +328,13 @@ void OutputManager::run(const util::TimeStamp& timestamp) return; } + EKAT_REQUIRE_MSG (timestamp<=m_output_control.next_write_ts, + "Error! The input timestamp is past the next scheduled write timestamp.\n" + " - current time stamp : " + timestamp.to_string() + "\n" + " - next write time stamp: " + m_output_control.next_write_ts.to_stirng() + "\n" + "The most likely cause is an output frequency that is faster than the atm timestep.\n" + "Try to increase 'Frequency' and/or 'frequency_units' in your output yaml file.\n"); + // Update counters ++m_output_control.nsamples_since_last_write; ++m_checkpoint_control.nsamples_since_last_write; From d0369c9a21a6d10a39bea0a674b68dba4a14a842 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Thu, 25 Jul 2024 12:17:34 -0700 Subject: [PATCH 206/477] Add support for ne512pg2 bi-grid --- cime_config/config_grids.xml | 46 +++++++++++++++++++ .../bld/namelist_files/namelist_defaults.xml | 2 + .../namelist_files/namelist_definition.xml | 2 +- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 7c8eb0ff75bd..7c3ebe83b0b0 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -1683,6 +1683,26 @@ oRRS18to6v3 + + ne512np4.pg2 + ne512np4.pg2 + ICOS10 + r0125 + null + null + ICOS10 + + + + ne512np4.pg2 + ne512np4.pg2 + ne512np4.pg2 + r0125 + null + null + ICOS10 + + ne512np4 360x720cru @@ -2894,6 +2914,8 @@ 1 $DIN_LOC_ROOT/share/domains/domain.lnd.ne512pg2_oRRS18to6v3.200212.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne512pg2_oRRS18to6v3.200212.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne512pg2_ICOS10.240602.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne512pg2_ICOS10.240602.nc ne512np4.pg2 is Spectral Elem 6km grid w/ 2x2 FV physics grid per element: @@ -3934,6 +3956,11 @@ cpl/gridmaps/ne256pg2/map_ne256pg2_to_r0125_bilin.200212.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_oRRS18to6v3_nco.200212.nc @@ -3943,6 +3970,15 @@ cpl/gridmaps/ne512pg2/map_oRRS18to6v3_to_ne512pg2_nco.200212.nc + + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ICOS10_to_ne512pg2_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ICOS10_to_ne512pg2_nco_c240531.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.200212.nc @@ -4717,6 +4753,11 @@ cpl/gridmaps/ne256pg2/map_r0125_to_ne256pg2_mono.200212.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_r0125_to_ne512pg2_mono.c20240625.nc + + cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc @@ -4797,6 +4838,11 @@ lnd/clm2/mappingdata/maps/ne240np4/map_0.5x0.5_nomask_to_ne240np4_nomask_aave_da_c121019.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_r0125_to_ne512pg2_mono.c20240625.nc + + cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 44cf85c73f3f..7ae61a1d4893 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -306,6 +306,8 @@ lnd/clm2/surfdata_map/surfdata_ne120pg2_simyr2010_c230119.nc lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr1850_c240131.nc lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr2010_c230207.nc + +lnd/clm2/surfdata_map/surfdata_ne512pg2_simyr2010_c240522.nc lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index ec3a74cbd965..72c7b1c3d654 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -1409,7 +1409,7 @@ CLM run type. +"512x1024,360x720cru,128x256,64x128,48x96,32x64,8x16,94x192,0.23x0.31,0.9x1.25,1.9x2.5,2.5x3.33,4x5,10x15,5x5_amazon,1x1_tropicAtl,1x1_camdenNJ,1x1_vancouverCAN,1x1_mexicocityMEX,1x1_asphaltjungleNJ,1x1_brazil,1x1_urbanc_alpha,1x1_numaIA,1x1_smallvilleIA,0.1x0.1,0.5x0.5,3x3min,5x5min,10x10min,0.33x0.33,ne4np4,ne4np4.pg2,ne11np4,ne16np4,ne30np4,ne30np4.pg2,ne60np4,ne120np4,ne120np4.pg2,ne240np4,ne256np4,ne256np4.pg2,ne512np4.pg2,ne1024np4,ne1024np4.pg2,1km-merge-10min,ne0np4_arm_x8v3_lowcon,ne0np4_conus_x4v1_lowcon,ne0np4_enax4v1,ne0np4_twpx4v1,r2,r05,r0125,NLDAS,ne0np4_northamericax4v1.pg2,ne0np4_arcticx4v1.pg2,r025"> Horizontal resolutions Note: 0.1x0.1, 0.5x0.5, 5x5min, 10x10min, 3x3min and 0.33x0.33 are only used for CLM tools From e5845eaaf8d2bb1cd07c0223c0848c76cc21617f Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 10:20:20 -0600 Subject: [PATCH 207/477] Fix how IOP reads time --- .../src/share/iop/intensive_observation_period.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 6f8474f8d85e..32b120c57002 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -248,15 +248,17 @@ initialize_iop_file(const util::TimeStamp& run_t0, else if (scorpio::has_dim(iop_file, "tsec")) time_dimname = "tsec"; else EKAT_ERROR_MSG("Error! No valid dimension for tsec in "+iop_file+".\n"); - const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); - m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); - scorpio::read_var(iop_file,"tsec",m_time_info.iop_file_times_in_sec.data()); - - // From now on, when we read vars, "time" must be treated as unlimited, to avoid issues + // When we read vars, "time" must be treated as unlimited, to avoid issues if (not scorpio::is_dim_unlimited(iop_file,time_dimname)) { scorpio::pretend_dim_is_unlimited(iop_file,time_dimname); } + const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); + m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); + for (int t=0; t Date: Thu, 25 Jul 2024 17:21:20 -0700 Subject: [PATCH 208/477] reduce the number of compsets down to one and make use of testmods to drive the tests of various cases --- cime_config/tests.py | 4 ++-- components/eamxx/cime_config/config_compsets.xml | 12 ------------ .../cime_config/namelist_defaults_scream.xml | 14 -------------- .../scream/dpxx/arm97/shell_commands | 15 +++++++++++++++ .../scream/dpxx/dycomsrf01/shell_commands | 15 +++++++++++++++ 5 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands diff --git a/cime_config/tests.py b/cime_config/tests.py index 275754e26a40..9c17744d0e64 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -657,8 +657,8 @@ "time" : "01:00:00", # each test 225 phys cols, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General", - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-ARM97", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-arm97", "ERS_P16_Ln22.ne30_ne30.F2000-SCREAMv1-DP-RCE", ) }, diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index e8c94534b919..c41bdb870ec7 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -80,12 +80,6 @@ Experimental, under development - - F2010-SCREAMv1-DP-ARM97 - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%ARM97 - Experimental, under development - - F2000-SCREAMv1-DP-RCE 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx @@ -98,8 +92,6 @@ 2016-08-01 2020-01-20 - 1999-07-10 - 1997-06-23 2000-01-01 @@ -150,16 +142,12 @@ - 31.5 - 36.605 0.0 - 238.5 - 262.515 0.0 diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index f1264b8b4e08..6b89e1fe7a3d 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -594,25 +594,11 @@ be lost if SCREAM_HACK_XML is not enabled. false - - ${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc - 36.605 - 262.515 - true - true - ${DIN_LOC_ROOT}/atm/cam/scam/iop/RCE_iopfile_4scam.nc 0.0 0.0 - - ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc - 31.5 - 238.5 - true - true - diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands new file mode 100644 index 000000000000..bda1791d66d7 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands @@ -0,0 +1,15 @@ +# Sets up DPxx case to test ARM97 (continental deep convection) +# case over land using prescribed surface fluxes and nudged winds + +./xmlchange RUN_STARTDATE="1997-06-23" +./xmlchange PTS_LAT=36.605 +./xmlchange PTS_LON=262.515 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file=${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=36.605 -b +$ATMCHANGE target_longitude=262.515 -b +$ATMCHANGE iop_nudge_uv=true -b +$ATMCHANGE iop_srf_prop=true -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands new file mode 100644 index 000000000000..58c235807872 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands @@ -0,0 +1,15 @@ +# Sets up DPxx case to test DYCOMSrf01 (marine stratocumulus) +# case over ocean using prescribed surface fluxes + +./xmlchange RUN_STARTDATE="1999-07-10" +./xmlchange PTS_LAT=31.5 +./xmlchange PTS_LON=238.5 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file=${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=31.5 -b +$ATMCHANGE target_longitude=238.5 -b +$ATMCHANGE iop_dosubsidence=true -b +$ATMCHANGE iop_srf_prop=true -b From e74604f2bed779b7a2439d9fa0fc0d4196605f90 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 25 Jul 2024 22:24:07 -0400 Subject: [PATCH 209/477] remove sunlit sync to host --- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 15de175da67d..8090a9300fd9 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -109,7 +109,6 @@ TEST_CASE("aodvis") { // Check result tau.sync_to_host(); - sunlit.sync_to_host(); diag->get_diagnostic().sync_to_host(); const auto tau_h = tau.get_view(); From 2ed55ace931870c0e12adfb7c50c94cfdcffd28c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Jul 2024 12:11:20 -0600 Subject: [PATCH 210/477] EAMxx: fix typo --- components/eamxx/src/share/io/scream_output_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 539204c66a1b..ccec5f516fd5 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -331,7 +331,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) EKAT_REQUIRE_MSG (timestamp<=m_output_control.next_write_ts, "Error! The input timestamp is past the next scheduled write timestamp.\n" " - current time stamp : " + timestamp.to_string() + "\n" - " - next write time stamp: " + m_output_control.next_write_ts.to_stirng() + "\n" + " - next write time stamp: " + m_output_control.next_write_ts.to_string() + "\n" "The most likely cause is an output frequency that is faster than the atm timestep.\n" "Try to increase 'Frequency' and/or 'frequency_units' in your output yaml file.\n"); From 59f9a9be68570796fc8e9c8dd9a0d0025bed728e Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 26 Jul 2024 13:04:18 -0600 Subject: [PATCH 211/477] Remove overuse of templates --- .../rrtmgp/scream_rrtmgp_interface.hpp | 234 ++++++++---------- 1 file changed, 109 insertions(+), 125 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index c82d9e38a3b6..0bceb1282e5f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -175,21 +175,40 @@ using hview_t = Kokkos::View; using pool_t = conv::MemPoolSingleton; +using real1dk = view_t; +using real2dk = view_t; +using real3dk = view_t; +using creal1dk = view_t; +using creal2dk = view_t; +using creal3dk = view_t; +using int1dk = view_t; +using int3dk = view_t; + +using gas_optics_t = GasOpticsRRTMGPK; +using cloud_optics_t = CloudOpticsK; +using gas_concs_t = GasConcsK; +using fluxes_t = FluxesBybandK; +using fluxes_broadband_t = FluxesBroadbandK; +using optical_props_t = OpticalPropsK; +using optical_props1_t = OpticalProps1sclK; +using optical_props2_t = OpticalProps2strK; +using source_func_t = SourceFuncLWK; + /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -static inline GasOpticsRRTMGPK k_dist_sw_k; -static inline GasOpticsRRTMGPK k_dist_lw_k; +static inline gas_optics_t k_dist_sw_k; +static inline gas_optics_t k_dist_lw_k; /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -static inline CloudOpticsK cloud_optics_sw_k; -static inline CloudOpticsK cloud_optics_lw_k; +static inline cloud_optics_t cloud_optics_sw_k; +static inline cloud_optics_t cloud_optics_lw_k; /* * Flag to indicate whether or not we have initialized RRTMGP @@ -200,7 +219,7 @@ static inline bool initialized_k = false; * Initialize data for RRTMGP driver */ static void rrtmgp_initialize( - GasConcsK &gas_concs, + const gas_concs_t &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger) @@ -238,13 +257,11 @@ static void rrtmgp_initialize( /* * Compute band-by-band surface albedos from broadband albedos. */ -template static void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, - SfcAlbDirVisT &sfc_alb_dir_vis, SfcAlbDirNirT &sfc_alb_dir_nir, - SfcAlbDifVisT &sfc_alb_dif_vis, SfcAlbDifNirT &sfc_alb_dif_nir, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif) + const creal1dk &sfc_alb_dir_vis, const creal1dk &sfc_alb_dir_nir, + const creal1dk &sfc_alb_dif_vis, const creal1dk &sfc_alb_dif_nir, + const real2dk &sfc_alb_dir, const real2dk &sfc_alb_dif) { EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); @@ -289,13 +306,11 @@ static void compute_band_by_band_surface_albedos( /* * Compute broadband visible/UV and near-infrared surface fluxes. */ -template static void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, - SwBndFluxDirT &sw_bnd_flux_dir , SwBndFluxDifT &sw_bnd_flux_dif , - SfcFluxDirVisT &sfc_flux_dir_vis, SfcFluxDirNirT &sfc_flux_dir_nir, - SfcFluxDifVisT &sfc_flux_dif_vis, SfcFluxDifNirT &sfc_flux_dif_nir) + const real3dk &sw_bnd_flux_dir , const real3dk &sw_bnd_flux_dif , + const real1dk &sfc_flux_dir_vis, const real1dk &sfc_flux_dir_nir, + const real1dk &sfc_flux_dif_vis, const real1dk &sfc_flux_dif_nir) { // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when @@ -349,42 +364,26 @@ static void compute_broadband_surface_fluxes( * The input logger is in charge of outputing info to * screen and/or to file (or neither), depending on how it was set up. */ -template static void rrtmgp_main( const int ncol, const int nlay, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei, CldfracT &cldfrac, - AerTauSwT &aer_tau_sw, AerSsaSwT &aer_ssa_sw, AerAsmSwT &aer_asm_sw, AerTauLwT &aer_tau_lw, - CldTauSwBndT &cld_tau_sw_bnd, CldTauLwBndT &cld_tau_lw_bnd, - CldTauSwGptT &cld_tau_sw_gpt, CldTauLwGptT &cld_tau_lw_gpt, - SwFluxUpT &sw_flux_up, SwFluxDnT &sw_flux_dn, SwFluxDnDirT &sw_flux_dn_dir, - LwFluxUpT &lw_flux_up, LwFluxDnT &lw_flux_dn, - SwClnclrskyFluxUpT &sw_clnclrsky_flux_up, SwClnclrskyFluxDnT &sw_clnclrsky_flux_dn, SwClnclrskyFluxDnDirT &sw_clnclrsky_flux_dn_dir, - SwClrskyFluxUpT &sw_clrsky_flux_up, SwClrskyFluxDnT &sw_clrsky_flux_dn, SwClrskyFluxDnDirT &sw_clrsky_flux_dn_dir, - SwClnskyFluxUpT &sw_clnsky_flux_up, SwClnskyFluxDnT &sw_clnsky_flux_dn, SwClnskyFluxDnDirT &sw_clnsky_flux_dn_dir, - LwClnclrskyFluxUpT &lw_clnclrsky_flux_up, LwClnclrskyFluxDnT &lw_clnclrsky_flux_dn, - LwClrskyFluxUpT &lw_clrsky_flux_up, LwClrskyFluxDnT &lw_clrsky_flux_dn, - LwClnskyFluxUpT &lw_clnsky_flux_up, LwClnskyFluxDnT &lw_clnsky_flux_dn, - SwBndFluxUpT &sw_bnd_flux_up, SwBndFluxDnT &sw_bnd_flux_dn, SwBndFluxDnDirT &sw_bnd_flux_dn_dir, - LwBndFluxUpT &lw_bnd_flux_up, LwBndFluxDnT &lw_bnd_flux_dn, - const RealT tsi_scaling, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei, const real2dk &cldfrac, + const real3dk &aer_tau_sw, const real3dk &aer_ssa_sw, const real3dk &aer_asm_sw, const real3dk &aer_tau_lw, + const real3dk &cld_tau_sw_bnd, const real3dk &cld_tau_lw_bnd, + const real3dk &cld_tau_sw_gpt, const real3dk &cld_tau_lw_gpt, + const real2dk &sw_flux_up, const real2dk &sw_flux_dn, const real2dk &sw_flux_dn_dir, + const real2dk &lw_flux_up, const real2dk &lw_flux_dn, + const real2dk &sw_clnclrsky_flux_up, const real2dk &sw_clnclrsky_flux_dn, const real2dk &sw_clnclrsky_flux_dn_dir, + const real2dk &sw_clrsky_flux_up, const real2dk &sw_clrsky_flux_dn, const real2dk &sw_clrsky_flux_dn_dir, + const real2dk &sw_clnsky_flux_up, const real2dk &sw_clnsky_flux_dn, const real2dk &sw_clnsky_flux_dn_dir, + const real2dk &lw_clnclrsky_flux_up, const real2dk &lw_clnclrsky_flux_dn, + const real2dk &lw_clrsky_flux_up, const real2dk &lw_clrsky_flux_dn, + const real2dk &lw_clnsky_flux_up, const real2dk &lw_clnsky_flux_dn, + const real3dk &sw_bnd_flux_up, const real3dk &sw_bnd_flux_dn, const real3dk &sw_bnd_flux_dn_dir, + const real3dk &lw_bnd_flux_up, const real3dk &lw_bnd_flux_dn, + const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) { @@ -404,7 +403,7 @@ static void rrtmgp_main( #endif // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; + fluxes_t fluxes_sw; fluxes_sw.flux_up = sw_flux_up; fluxes_sw.flux_dn = sw_flux_dn; fluxes_sw.flux_dn_dir = sw_flux_dn_dir; @@ -412,37 +411,37 @@ static void rrtmgp_main( fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; + fluxes_broadband_t clnclrsky_fluxes_sw; clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; + fluxes_broadband_t clrsky_fluxes_sw; clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; + fluxes_broadband_t clnsky_fluxes_sw; clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; + fluxes_t fluxes_lw; fluxes_lw.flux_up = lw_flux_up; fluxes_lw.flux_dn = lw_flux_dn; fluxes_lw.bnd_flux_up = lw_bnd_flux_up; fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; + fluxes_broadband_t clnclrsky_fluxes_lw; clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; + fluxes_broadband_t clrsky_fluxes_lw; clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; + fluxes_broadband_t clnsky_fluxes_lw; clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; @@ -450,8 +449,8 @@ static void rrtmgp_main( auto nlwbands = k_dist_lw_k.get_nband(); // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; + optical_props2_t aerosol_sw; + optical_props1_t aerosol_lw; aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); aerosol_sw.alloc_2str(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { @@ -476,8 +475,8 @@ static void rrtmgp_main( #endif // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + optical_props2_t clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + optical_props1_t clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); @@ -549,17 +548,15 @@ static void rrtmgp_finalize() /* * Shortwave driver (called by rrtmgp_main) */ -template static void rrtmgp_sw( const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const RealT tsi_scaling, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + optical_props2_t &aerosol, optical_props2_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, + const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { @@ -647,7 +644,7 @@ static void rrtmgp_sw( // Subset gases auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; + gas_concs_t gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { auto vmr_day = view_t("vmr_day", nday, nlay); @@ -660,7 +657,7 @@ static void rrtmgp_sw( } // Subset aerosol optics - OpticalProps2strK aerosol_day; + optical_props2_t aerosol_day; aerosol_day.init(k_dist.get_band_lims_wavenumber()); aerosol_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { @@ -671,7 +668,7 @@ static void rrtmgp_sw( // Subset cloud optics // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; + optical_props2_t clouds_day; clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); clouds_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { @@ -697,7 +694,7 @@ static void rrtmgp_sw( auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; + fluxes_t fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; fluxes_day.flux_dn_dir = flux_dn_dir_day; @@ -706,10 +703,10 @@ static void rrtmgp_sw( fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; // Allocate space for optical properties - OpticalProps2strK optics; + optical_props2_t optics; optics.alloc_2str(nday, nlay, k_dist); - OpticalProps2strK optics_no_aerosols; + optical_props2_t optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_2str(nday, nlay, k_dist); @@ -811,14 +808,13 @@ static void rrtmgp_sw( /* * Longwave driver (called by rrtmgp_main) */ -template static void rrtmgp_lw( const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + optical_props1_t &aerosol, optical_props1_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Problem size @@ -856,16 +852,16 @@ static void rrtmgp_lw( }); // Allocate space for optical properties - OpticalProps1sclK optics; + optical_props1_t optics; optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; + optical_props1_t optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); } // Boundary conditions - SourceFuncLWK lw_sources; + source_func_t lw_sources; lw_sources.alloc(ncol, nlay, k_dist); view_t t_sfc ("t_sfc" ,ncol); view_t emis_sfc("emis_sfc",nbnd,ncol); @@ -950,12 +946,10 @@ static void rrtmgp_lw( /* * Return a subcolumn mask consistent with a specified overlap assumption */ -template -static view_t get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, - CldfT &cldf, const int overlap_option, SeedsT &seeds) +static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds) { // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = view_t("subcolumn_mask", ncol, nlay, ngpt); + int3dk subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); // Subcolumn generators are a means for producing a variable x(i,j,k), where // @@ -1029,10 +1023,9 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i /* * Compute cloud area from 3d subcol cloud property */ -template static void compute_cloud_area( - int ncol, int nlay, int ngpt, RealT pmin, RealT pmax, - const PmidT& pmid, const CldTauGptT& cld_tau_gpt, CldAreaT& cld_area) + int ncol, int nlay, int ngpt, Real pmin, Real pmax, + const creal2dk& pmid, const real3dk& cld_tau_gpt, const real1dk& cld_area) { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 @@ -1058,20 +1051,15 @@ static void compute_cloud_area( /* * Return select cloud-top diagnostics following AeroCom recommendation */ -template static void compute_aerocom_cloudtop( - int ncol, int nlay, const TmidT &tmid, const PmidT &pmid, - const PdelT &p_del, const ZdelT &z_del, const QcT &qc, - const QiT &qi, const RelT &rel, const ReiT &rei, - const CldfracTotT &cldfrac_tot, const NcT &nc, - TmidAtCldtopT &T_mid_at_cldtop, PmidAtCldtopT &p_mid_at_cldtop, - CldfracIceAtCldtopT &cldfrac_ice_at_cldtop, CldfracLiqAtCldtopT &cldfrac_liq_at_cldtop, - CldfracTotAtCldtopT &cldfrac_tot_at_cldtop, CdncAtCldtopT &cdnc_at_cldtop, - EffRadiusQcAtCldtopT &eff_radius_qc_at_cldtop, EffRadiusQiAtCldTopT &eff_radius_qi_at_cldtop) + int ncol, int nlay, const creal2dk &tmid, const creal2dk &pmid, + const creal2dk &p_del, const real2dk &z_del, const creal2dk &qc, + const creal2dk &qi, const creal2dk &rel, const creal2dk &rei, + const real2dk &cldfrac_tot, const creal2dk &nc, + const real1dk &T_mid_at_cldtop, const real1dk &p_mid_at_cldtop, + const real1dk &cldfrac_ice_at_cldtop, const real1dk &cldfrac_liq_at_cldtop, + const real1dk &cldfrac_tot_at_cldtop, const real1dk &cdnc_at_cldtop, + const real1dk &eff_radius_qc_at_cldtop, const real1dk &eff_radius_qi_at_cldtop) { /* The goal of this routine is to calculate properties at cloud top * based on the AeroCom recommendation. See reference for routine @@ -1217,7 +1205,7 @@ static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, O } -static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) +static int get_wavelength_index(optical_props_t &kdist, RealT wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -1251,14 +1239,13 @@ static inline int get_wavelength_index_lw_k(RealT wavelength) return get_wavelength_index(k_dist_lw_k, wavelength); } -template -static OpticalProps2strK get_cloud_optics_sw( +static optical_props2_t get_cloud_optics_sw( const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { // Initialize optics - OpticalProps2strK clouds; + optical_props2_t clouds; clouds.init(kdist.get_band_lims_wavenumber()); clouds.alloc_2str(ncol, nlay); @@ -1278,14 +1265,13 @@ static OpticalProps2strK get_cloud_optics_sw( return clouds; } -template -static OpticalProps1sclK get_cloud_optics_lw( +static optical_props1_t get_cloud_optics_lw( const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { // Initialize optics - OpticalProps1sclK clouds; + optical_props1_t clouds; clouds.init(kdist.get_band_lims_wavenumber()); clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! @@ -1305,12 +1291,11 @@ static OpticalProps1sclK get_cloud_optics_lw( return clouds; } -template -static OpticalProps2strK get_subsampled_clouds( +static optical_props2_t get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { + optical_props2_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { // Initialized subsampled optics - OpticalProps2strK subsampled_optics; + optical_props2_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_2str(ncol, nlay); // Check that we do not have clouds with no optical properties; this would get corrected @@ -1356,13 +1341,12 @@ static OpticalProps2strK get_subsampled_clouds( return subsampled_optics; } -template -static OpticalProps1sclK get_subsampled_clouds( +static optical_props1_t get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, - CldT &cld, PlayT &p_lay) { + optical_props1_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { + // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; + optical_props1_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_1scl(ncol, nlay); // Check that we do not have clouds with no optical properties; this would get corrected From de1862c61a0824480c3ff21d18f8f5f31a1023e9 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 26 Jul 2024 15:35:47 -0500 Subject: [PATCH 212/477] Downgrade libunwind for machine frontier-scream-gpu Downgrade libunwind from 1.6.2 to 1.5.0 to prevent ums002/default, which is required to load libunwind/1.6.2, from resetting rocm/5.4.0 back to rocm/5.3.0. --- cime_config/machines/config_machines.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3617bb88f9ec..7cd34fdb0247 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1437,9 +1437,7 @@ PrgEnv-cray craype-accel-amd-gfx90a rocm/5.4.0 - ums/default - ums002/default - libunwind/1.6.2 + libunwind/1.5.0 cce/15.0.1 From d4d7e0e15f457223ccea74d9e661d7d555f29451 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 26 Jul 2024 15:56:28 -0500 Subject: [PATCH 213/477] Run hipInit prior to MPI_Init in subroutine cime_pre_init1() This workaround is used to avoid occasional segfaults during MPI_Init. Not required for ROCm 5.5.1 or newer releases. --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 9 +++++++++ .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 6 ++++++ components/eamxx/src/mct_coupling/scream_f2c_mod.F90 | 3 +++ driver-mct/main/cime_comp_mod.F90 | 2 ++ 4 files changed, 20 insertions(+) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 34bbbedcc5c4..bc79f6c11a7a 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -25,6 +25,7 @@ module atm_comp_mct ! Public interfaces !-------------------------------------------------------------------------- + public :: atm_init_hip_mct public :: atm_init_mct public :: atm_run_mct public :: atm_final_mct @@ -46,6 +47,14 @@ module atm_comp_mct CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + !=============================================================================== + subroutine atm_init_hip_mct() + use scream_f2c_mod, only: scream_init_hip_atm + + call scream_init_hip_atm() + + end subroutine atm_init_hip_mct + !=============================================================================== subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use iso_c_binding, only: c_ptr, c_loc, c_int, c_char, c_bool diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 0bdf90eeb71c..99a7ff0d8240 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -21,6 +21,8 @@ #include "ekat/ekat_pack.hpp" #include "ekat/ekat_assert.hpp" +#include + // Anonymous namespace, for some utility functions namespace { @@ -202,6 +204,10 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor }); } +void scream_init_hip_atm () { + hipInit(0); +} + void scream_init_atm (const char* caseid, const char* hostname, const char* username) diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 51c3eca91230..25abb1495ba5 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -69,6 +69,9 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices integer(kind=c_int), intent(in) :: import_field_size, export_field_size end subroutine scream_setup_surface_coupling + subroutine scream_init_hip_atm () bind(c) + end subroutine scream_init_hip_atm + ! This subroutine performs completes the initialization of the atm instance. ! In particular, this routine must be called *after* scream_create_atm_instance, ! and *after* scream_setup_surface_coupling. diff --git a/driver-mct/main/cime_comp_mod.F90 b/driver-mct/main/cime_comp_mod.F90 index 2131d0c86844..8282a76109c8 100644 --- a/driver-mct/main/cime_comp_mod.F90 +++ b/driver-mct/main/cime_comp_mod.F90 @@ -714,6 +714,7 @@ module cime_comp_mod subroutine cime_pre_init1(esmf_log_option) use shr_pio_mod, only : shr_pio_init1, shr_pio_init2 use seq_comm_mct, only: num_inst_driver + use atm_comp_mct, only: atm_init_hip_mct !---------------------------------------------------------- !| Initialize MCT and MPI communicators and IO !---------------------------------------------------------- @@ -736,6 +737,7 @@ subroutine cime_pre_init1(esmf_log_option) beg_count = shr_sys_irtc(irtc_rate) + call atm_init_hip_mct() call mpi_init(ierr) call shr_mpi_chkerr(ierr,subname//' mpi_init') From 4411d44834873c7d988f4af4b47e2c54a7483133 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 26 Jul 2024 18:06:26 -0500 Subject: [PATCH 214/477] Add SCREAM_SYSTEM_WORKAROUND macro This macro is used to optionally run hipInit prior to MPI_Init to avoid occasional segfaults. It can be turned off for ROCm 5.5.1 or newer releases. --- .../cmake_macros/crayclang-scream_frontier-scream-gpu.cmake | 2 +- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 4 ++++ .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 4 ++++ components/eamxx/src/mct_coupling/scream_f2c_mod.F90 | 2 ++ driver-mct/main/cime_comp_mod.F90 | 4 ++++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index cc90b369244a..a5c89c3318ed 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -5,7 +5,7 @@ set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") -string(APPEND CPPDEFS " -DLINUX") +string(APPEND CPPDEFS " -DLINUX -DSCREAM_SYSTEM_WORKAROUND=1") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index bc79f6c11a7a..dc57d828bbdb 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -25,7 +25,9 @@ module atm_comp_mct ! Public interfaces !-------------------------------------------------------------------------- +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) public :: atm_init_hip_mct +#endif public :: atm_init_mct public :: atm_run_mct public :: atm_final_mct @@ -47,6 +49,7 @@ module atm_comp_mct CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) !=============================================================================== subroutine atm_init_hip_mct() use scream_f2c_mod, only: scream_init_hip_atm @@ -54,6 +57,7 @@ subroutine atm_init_hip_mct() call scream_init_hip_atm() end subroutine atm_init_hip_mct +#endif !=============================================================================== subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 99a7ff0d8240..e9129e6fcea3 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -21,7 +21,9 @@ #include "ekat/ekat_pack.hpp" #include "ekat/ekat_assert.hpp" +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) #include +#endif // Anonymous namespace, for some utility functions namespace { @@ -204,9 +206,11 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor }); } +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) void scream_init_hip_atm () { hipInit(0); } +#endif void scream_init_atm (const char* caseid, const char* hostname, diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 25abb1495ba5..f7a0bc9c3794 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -69,8 +69,10 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices integer(kind=c_int), intent(in) :: import_field_size, export_field_size end subroutine scream_setup_surface_coupling +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) subroutine scream_init_hip_atm () bind(c) end subroutine scream_init_hip_atm +#endif ! This subroutine performs completes the initialization of the atm instance. ! In particular, this routine must be called *after* scream_create_atm_instance, diff --git a/driver-mct/main/cime_comp_mod.F90 b/driver-mct/main/cime_comp_mod.F90 index 8282a76109c8..7cf442b9413d 100644 --- a/driver-mct/main/cime_comp_mod.F90 +++ b/driver-mct/main/cime_comp_mod.F90 @@ -714,7 +714,9 @@ module cime_comp_mod subroutine cime_pre_init1(esmf_log_option) use shr_pio_mod, only : shr_pio_init1, shr_pio_init2 use seq_comm_mct, only: num_inst_driver +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) use atm_comp_mct, only: atm_init_hip_mct +#endif !---------------------------------------------------------- !| Initialize MCT and MPI communicators and IO !---------------------------------------------------------- @@ -737,7 +739,9 @@ subroutine cime_pre_init1(esmf_log_option) beg_count = shr_sys_irtc(irtc_rate) +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) call atm_init_hip_mct() +#endif call mpi_init(ierr) call shr_mpi_chkerr(ierr,subname//' mpi_init') From 1f77bf0e48729dc1f06fad26933fa885a9e1e0d2 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 29 Jul 2024 10:55:32 -0600 Subject: [PATCH 215/477] Changes due to code review feedback. --- .../eamxx_mam_wetscav_process_interface.cpp | 68 +++++++++++++------ .../eamxx_mam_wetscav_process_interface.hpp | 36 ++++++++-- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index b598fa6d7483..2a81c1a9c427 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -29,7 +29,6 @@ void MAMWetscav::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. auto q_unit = kg / kg; - auto dqdt_unit = kg / kg / s; auto n_unit = 1 / kg; // units of number mixing ratios of tracers m_grid = grids_manager->get_grid("Physics"); @@ -206,7 +205,7 @@ void MAMWetscav::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- - static constexpr auto m3 = m2 * m; + static constexpr auto m3 = m * m * m; // Aerosol dry particle diameter [m] add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); @@ -357,6 +356,44 @@ void MAMWetscav::initialize_impl(const RunType run_type) { const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now + + sh_frac_ = view_2d("sh_frac", ncol_, nlev_); + Kokkos::deep_copy(sh_frac_, 0); + + // Deep convective cloud fraction [fraction] + dp_frac_ = view_2d("dp_frac", ncol_, nlev_); + Kokkos::deep_copy(dp_frac_, 0); + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + evapcsh_ = view_2d("evapcsh", ncol_, nlev_); + Kokkos::deep_copy(evapcsh_, 0); + + // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] + evapcdp_ = view_2d("evapcdp", ncol_, nlev_); + Kokkos::deep_copy(evapcdp_, 0); + + // Rain production, shallow convection [kg/kg/s] + rprdsh_ = view_2d("rprdsh", ncol_, nlev_); + Kokkos::deep_copy(rprdsh_, 0); + + // Rain production, deep convection [kg/kg/s] + rprddp_ = view_2d("rprddp", ncol_, nlev_); + Kokkos::deep_copy(rprddp_, 0); + + // In cloud water mixing ratio, deep convection + icwmrdp_ = view_2d("icwmrdp", ncol_, nlev_); + Kokkos::deep_copy(icwmrdp_, 0); + + // In cloud water mixing ratio, shallow convection + icwmrsh_ = view_2d("icwmrsh", ncol_, nlev_); + Kokkos::deep_copy(icwmrsh_, 0); + + // Detraining cld H20 from deep convection [kg/kg/s] + dlf_ = view_2d("dlf", ncol_, nlev_); + Kokkos::deep_copy(dlf_, 0); + //--------------------------------------------------------------------------------- // Setup preprocessing and post processing //--------------------------------------------------------------------------------- @@ -394,40 +431,31 @@ void MAMWetscav::run_impl(const double dt) { // TODO: Following variables are from convective parameterization (not // implemented yet in EAMxx), so should be zero for now - auto sh_frac = view_2d("sh_frac", ncol_, nlev_); - Kokkos::deep_copy(sh_frac, 0); + auto sh_frac = sh_frac_; // Deep convective cloud fraction [fraction] - auto dp_frac = view_2d("dp_frac", ncol_, nlev_); - Kokkos::deep_copy(dp_frac, 0); + auto dp_frac = dp_frac_; // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] - auto evapcsh = view_2d("evapcsh", ncol_, nlev_); - Kokkos::deep_copy(evapcsh, 0); + auto evapcsh = evapcsh_; // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] - auto evapcdp = view_2d("evapcdp", ncol_, nlev_); - Kokkos::deep_copy(evapcdp, 0); + auto evapcdp = evapcdp_; // Rain production, shallow convection [kg/kg/s] - auto rprdsh = view_2d("rprdsh", ncol_, nlev_); - Kokkos::deep_copy(rprdsh, 0); + auto rprdsh = rprdsh_; // Rain production, deep convection [kg/kg/s] - auto rprddp = view_2d("rprddp", ncol_, nlev_); - Kokkos::deep_copy(rprddp, 0); + auto rprddp = rprddp_; // In cloud water mixing ratio, deep convection - auto icwmrdp = view_2d("icwmrdp", ncol_, nlev_); - Kokkos::deep_copy(icwmrdp, 0); + auto icwmrdp = icwmrdp_; // In cloud water mixing ratio, shallow convection - auto icwmrsh = view_2d("icwmrsh", ncol_, nlev_); - Kokkos::deep_copy(icwmrsh, 0); + auto icwmrsh = icwmrsh_; // Detraining cld H20 from deep convection [kg/kg/s] - auto dlf = view_2d("dlf", ncol_, nlev_); - Kokkos::deep_copy(dlf, 0); + auto dlf = dlf_; //----------- Variables from macrophysics scheme ------------- // Total cloud fraction diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index 8bd81115d8a3..ac811cab34d1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -40,10 +40,10 @@ class MAMWetscav : public scream::AtmosphereProcess { MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The type of subcomponent - AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type() const override { return AtmosphereProcessType::Physics; } // The name of the subcomponent - std::string name() const { return "mam4_wetscav"; } + std::string name() const override { return "mam4_wetscav"; } // Set the grid and input output variables void set_grids( @@ -52,7 +52,7 @@ class MAMWetscav : public scream::AtmosphereProcess { // management of common atm process memory // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels - size_t requested_buffer_size_in_bytes() const { + size_t requested_buffer_size_in_bytes() const override { return mam_coupling::buffer_size(ncol_, nlev_); } void init_buffers(const ATMBufferManager &buffer_manager) override; @@ -64,7 +64,7 @@ class MAMWetscav : public scream::AtmosphereProcess { void run_impl(const double dt) override; // Finalize - void finalize_impl(){/*Do nothing*/}; + void finalize_impl() override {/*Do nothing*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. @@ -168,6 +168,34 @@ class MAMWetscav : public scream::AtmosphereProcess { // Work arrays view_2d work_; + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now + + view_2d sh_frac_; + + // Deep convective cloud fraction [fraction] + view_2d dp_frac_; + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + view_2d evapcsh_; + + view_2d evapcdp_; + + // Rain production, shallow convection [kg/kg/s] + view_2d rprdsh_; + + // Rain production, deep convection [kg/kg/s] + view_2d rprddp_; + + // In cloud water mixing ratio, deep convection + view_2d icwmrdp_; + + // In cloud water mixing ratio, shallow convection + view_2d icwmrsh_; + + // Detraining cld H20 from deep convection [kg/kg/s] + view_2d dlf_; + // Aerosol states mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; From 2890ec1f87bdf12f910d28667e8a35a822e7df89 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 29 Jul 2024 11:30:38 -0600 Subject: [PATCH 216/477] Changes due to code review feedback. --- .../physics/mam/eamxx_mam_wetscav_process_interface.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index ac811cab34d1..9d1dc20bc88f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -10,14 +10,6 @@ // For component name #include -#ifndef KOKKOS_ENABLE_CUDA -#define protected_except_cuda public -#define private_except_cuda public -#else -#define protected_except_cuda protected -#define private_except_cuda private -#endif - namespace scream { /* From 846e5044429caf921a74c9b49e194e004b76115a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 29 Jul 2024 15:00:28 -0600 Subject: [PATCH 217/477] EAMxx: fix check for io --- components/eamxx/src/share/io/scream_output_manager.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index ccec5f516fd5..13e2e97101b1 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -328,7 +328,11 @@ void OutputManager::run(const util::TimeStamp& timestamp) return; } - EKAT_REQUIRE_MSG (timestamp<=m_output_control.next_write_ts, + // Ensure we did not go past the scheduled write time without hitting it + EKAT_REQUIRE_MSG ( + (m_output_control.frequency_units=="nsteps" + ? timestamp.get_num_steps()<=m_output_control.next_write_ts.get_num_steps() + : timestamp<=m_output_control.next_write_ts), "Error! The input timestamp is past the next scheduled write timestamp.\n" " - current time stamp : " + timestamp.to_string() + "\n" " - next write time stamp: " + m_output_control.next_write_ts.to_string() + "\n" From eabd0fed3fb21bce4e634d76b673e65cb78f7cf4 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 29 Jul 2024 16:27:32 -0600 Subject: [PATCH 218/477] Changes due to code review feedback. --- ...x_mam_dry_deposition_process_interface.cpp | 24 +++++++------------ .../eamxx/src/physics/mam/mam_coupling.hpp | 1 + 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 433c9d4f6063..4a9dcbf1eca6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -41,28 +41,22 @@ void MAMDryDep::set_grids( using namespace ShortFieldTagsNames; // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d{{COL}, {ncol_}}; + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - const FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + const FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + const FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); // layout for 2D (ncol, pcnst) constexpr int pcnst = mam4::aero_model::pcnst; - const FieldLayout scalar2d_pcnct = + const FieldLayout vector2d_pcnst = grid_->get_2d_vector_layout(pcnst, "num_phys_constants"); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables // at mid points - auto make_layout = [](const std::vector &extents, - const std::vector &names) { - std::vector tags(extents.size(), CMP); - return FieldLayout(tags, extents, names); - }; const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "num_modes", "lev"}); + const FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true, num_aero_modes, "num_modes"); using namespace ekat::units; @@ -150,10 +144,10 @@ void MAMDryDep::set_grids( //----------- Variables from other mam4xx processes ------------ // Geometric mean wet diameter for number distribution [m] - add_field("dgnumwet", scalar4d_mid, m, grid_name); + add_field("dgnumwet", vector3d_mid, m, grid_name); // Wet density of interstitial aerosol [kg/m3] - add_field("wetdens", scalar4d_mid, kg / m3, grid_name); + add_field("wetdens", vector3d_mid, kg / m3, grid_name); // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process @@ -203,11 +197,11 @@ void MAMDryDep::set_grids( // ------------------------------------------------------------- // FIXME: These are diagnostics, remove them from FM after initial evaluation // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] - add_field("deposition_flux_of_cloud_borne_aerosols", scalar2d_pcnct, + add_field("deposition_flux_of_cloud_borne_aerosols", vector2d_pcnst, 1 / m2 / s, grid_name); // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] add_field("deposition_flux_of_interstitial_aerosols", - scalar2d_pcnct, 1 / m2 / s, grid_name); + vector2d_pcnst, 1 / m2 / s, grid_name); } // set_grids // ================================================================ diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 1e13cece98d9..961ad675fbd1 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -606,6 +606,7 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); + // NOTE: we are not currently allowing surface topography: EKAT_KERNEL_ASSERT_MSG(dry_atm.z_surf == 0, "dry_atm.z_surf must be zero"); PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output From 405486e3e5420b72de8120ec0792b31ce7a5c128 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 29 Jul 2024 18:43:26 -0700 Subject: [PATCH 219/477] add pyatmproc list field methods --- .../src/python/libpyscream/pyatmproc.hpp | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 000a7d64c9a9..108edc2551b3 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -131,12 +131,32 @@ struct PyAtmProc { return pybind11::cast(missing); } - pybind11::list dump_fields() { - std::vector all_fields; - for (auto it : fields) { - all_fields.push_back(it.first); + pybind11::list list_fields(std::string ftype) { + std::vector fields_list; + for (const auto& field_pair : fields) { + const auto& field_identifier = field_pair.second.f.get_header().get_identifier(); + + if (ftype == "required" && ap->has_required_field(field_identifier)) { + fields_list.push_back(field_pair.first); + } else if (ftype == "computed" && ap->has_computed_field(field_identifier)) { + fields_list.push_back(field_pair.first); + } else if (ftype == "all") { + fields_list.push_back(field_pair.first); + } } - return pybind11::cast(all_fields); + return pybind11::cast(fields_list); + } + + pybind11::list list_all_fields() { + return list_fields("all"); + } + + pybind11::list list_required_fields() { + return list_fields("required"); + } + + pybind11::list list_computed_fields() { + return list_fields("computed"); } void setup_output (const std::string& yaml_file) { @@ -184,7 +204,9 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) .def("read_ic",&PyAtmProc::read_ic) - .def("dump_fields",&PyAtmProc::dump_fields); + .def("list_all_fields",&PyAtmProc::list_all_fields) + .def("list_required_fields",&PyAtmProc::list_required_fields) + .def("list_computed_fields",&PyAtmProc::list_computed_fields); } } // namespace scream From 8d49236cf761aae4e6d20ae768f1c8c87070f60c Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 29 Jul 2024 19:02:25 -0700 Subject: [PATCH 220/477] replace std::numeric_limits with kokkos norm_min --- .../shoc/impl/shoc_assumed_pdf_impl.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp index a407e83c2ee7..84e76c5941a1 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp @@ -9,6 +9,20 @@ namespace scream { namespace shoc { +/* +Add some functions to avoid cuda compilation warnings +TODO: move this to ekat or something +*/ +#ifdef KOKKOS_ENABLE_CUDA +KOKKOS_INLINE_FUNCTION constexpr Real safe_min() { + return Kokkos::Experimental::norm_min_v; +} +#else +KOKKOS_INLINE_FUNCTION constexpr Real safe_min() { + return std::numeric_limits::min(); +} +#endif + /* * Implementation of shoc shoc_assumed_pdf. Clients should NOT * #include this file, but include shoc_functions.hpp instead. @@ -317,7 +331,7 @@ void Functions::shoc_assumed_pdf( std_s1 = ekat::sqrt(ekat::max(0, ekat::square(cthl1)*thl2_1 + ekat::square(cqt1)*qw2_1 - 2*cthl1*sqrtthl2_1*cqt1*sqrtqw2_1*r_qwthl_1)); - const auto std_s1_not_small = std_s1 > std::sqrt(std::numeric_limits::min()) * 100; + const auto std_s1_not_small = std_s1 > std::sqrt(safe_min()) * 100; s1 = qw1_1-qs1*((1 + beta1*qw1_1)/(1 + beta1*qs1)); if (std_s1_not_small.any()) { C1.set(std_s1_not_small, sp(0.5)*(1 + ekat::erf(s1/(sqrt2*std_s1)))); @@ -357,7 +371,7 @@ void Functions::shoc_assumed_pdf( ekat::square(cthl2)*thl2_2 + ekat::square(cqt2)*qw2_2 - 2*cthl2*sqrtthl2_2*cqt2*sqrtqw2_2*r_qwthl_1))); s2.set(nequal, qw1_2-qs2*((1 + beta2*qw1_2)/(1 + beta2*qs2))); - const auto std_s2_not_small = std_s2 > std::sqrt(std::numeric_limits::min()) * 100; + const auto std_s2_not_small = std_s2 > std::sqrt(safe_min()) * 100; const auto nequal_std_s2_not_small = nequal && std_s2_not_small; if (nequal_std_s2_not_small.any()) { C2.set(nequal_std_s2_not_small, sp(0.5)*(1 + ekat::erf(s2/(sqrt2*std_s2)))); From dd4b09a565540e6bd20e908fbdf55644df845902 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 29 Jul 2024 19:23:53 -0700 Subject: [PATCH 221/477] update haero module --- externals/haero | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/haero b/externals/haero index 8e4c96ded375..0dff12afaf72 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit 8e4c96ded375bf03bb5e83b0779e45a3aec9659e +Subproject commit 0dff12afaf72f3dff7831676260812d41dd68683 From 8fd0eae1ad3d3ae389b10e1e685d4061714e256d Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 30 Jul 2024 08:47:05 -0600 Subject: [PATCH 222/477] Changes due to code review feedback. --- .../eamxx_mam_wetscav_process_interface.cpp | 38 ++++++------------- .../eamxx_mam_wetscav_process_interface.hpp | 12 +++++- .../mam/p3_mam4_wetscav/output.yaml | 4 -- .../mam/shoc_cldfrac_p3_wetscav/output.yaml | 4 -- .../single-process/mam/wet_scav/output.yaml | 4 -- 5 files changed, 23 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 2a81c1a9c427..81f13fd6ae9a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -47,10 +47,6 @@ void MAMWetscav::set_grids( // layout for 2D (1d horiz X 1d vertical) variables FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); - // layout for 3D (ncol, nmodes, nlevs) - FieldLayout scalar3d_mid_nmodes = - m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); - // layout for 2D (ncol, pcnst) FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "num_phys_constants"); @@ -205,22 +201,6 @@ void MAMWetscav::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- - static constexpr auto m3 = m * m * m; - - // Aerosol dry particle diameter [m] - add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); - - // Wet aerosol density [kg/m3] - add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); - - // Aerosol water [kg/kg] - add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); - - // Wet aerosol diameter [m] - add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); - - // Fraction of transported species that are insoluble [fraction] - add_field("fracis", scalar3d_mid, nondim, grid_name); // Aerosol wet deposition (interstitial) [kg/m2/s] add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); @@ -352,6 +332,14 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } + const int nmodes = mam4::AeroConfig::num_modes(); + + // Aerosol dry particle diameter [m] + dgncur_a_ = view_3d("dgncur_a", ncol_, nmodes, nlev_); + wetdens_ = view_3d("wetdens", ncol_, nmodes, nlev_); + qaerwat_ = view_3d("qaerwat", ncol_, nmodes, nlev_); + dgnumwet_ = view_3d("dgnumwet", ncol_, nmodes, nlev_); + // Allocate work array const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); @@ -475,12 +463,10 @@ void MAMWetscav::run_impl(const double dt) { const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); - const auto wet_geometric_mean_diameter_i = - get_field_out("dgnumwet").get_view(); - const auto dry_geometric_mean_diameter_i = - get_field_out("dgncur_a").get_view(); - const auto qaerwat = get_field_out("qaerwat").get_view(); - const auto wetdens = get_field_out("wetdens").get_view(); + const auto wet_geometric_mean_diameter_i = dgnumwet_; + const auto dry_geometric_mean_diameter_i = dgncur_a_; + const auto qaerwat = qaerwat_; + const auto wetdens = wetdens_; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index 9d1dc20bc88f..3c3dec6cc623 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -23,6 +23,7 @@ namespace scream { class MAMWetscav : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; using view_2d = typename KT::template view_2d; + using view_3d = typename KT::template view_3d; // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; @@ -160,9 +161,18 @@ class MAMWetscav : public scream::AtmosphereProcess { // Work arrays view_2d work_; + // Aerosol dry particle diameter [m] + // Dimensions: [cols, modes, levels] + view_3d dgncur_a_; + // Wet aerosol density [kg/m3] + view_3d wetdens_; + // Aerosol water [kg/kg] + view_3d qaerwat_; + // Wet aerosol diameter [m] + view_3d dgnumwet_; + // TODO: Following variables are from convective parameterization (not // implemented yet in EAMxx), so should be zero for now - view_2d sh_frac_; // Deep convective cloud fraction [fraction] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml index c26db8eb6814..049dcdad35a5 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -56,10 +56,6 @@ Field Names: - num_c4 - aerdepwetis - aerdepwetcw - - dgnumwet - - dgncur_a - - wetdens - - qaerwat output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml index 1ae53e6c1c30..c38bafaa65e3 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -90,10 +90,6 @@ Field Names: - num_c4 - aerdepwetis - aerdepwetcw - - dgnumwet - - dgncur_a - - wetdens - - qaerwat output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 9cd0a16bc414..75172823870e 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -7,10 +7,6 @@ Fields: Field Names: - aerdepwetis - aerdepwetcw - - dgnumwet - - dgncur_a - - wetdens - - qaerwat - bc_c1 - bc_c3 - bc_c4 From 99e46d6b768ea1422246f2e5a6998b49ce5d3737 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 30 Jul 2024 09:14:31 -0700 Subject: [PATCH 223/477] add test for COMBLE and various fixes --- cime_config/tests.py | 1 + .../scream/dpxx/arm97/shell_commands | 2 +- .../scream/dpxx/comble/shell_commands | 20 +++++++++++++++++++ .../scream/dpxx/dycomsrf01/shell_commands | 2 +- 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands diff --git a/cime_config/tests.py b/cime_config/tests.py index 9c17744d0e64..dd08de8c2540 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -659,6 +659,7 @@ "tests" : ( "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-dycomsrf01", "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-arm97", + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-comble", "ERS_P16_Ln22.ne30_ne30.F2000-SCREAMv1-DP-RCE", ) }, diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands index bda1791d66d7..4cba77f45f81 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands @@ -8,7 +8,7 @@ # Scripts location ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange -$ATMCHANGE iop_file=${DIN_LOC_ROOT}/atm/cam/scam/iop/ARM97_iopfile_4scam.nc -b +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/ARM97_iopfile_4scam.nc -b $ATMCHANGE target_latitude=36.605 -b $ATMCHANGE target_longitude=262.515 -b $ATMCHANGE iop_nudge_uv=true -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands new file mode 100644 index 000000000000..166e77b1f4bb --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -0,0 +1,20 @@ +# Sets up DPxx case to test COMBLE (cold air outbreak) +# lagrangian case over ocean with interactive surface fluxes and coriolis forcing applied to winds + +./xmlchange RUN_STARTDATE="2020-03-12" +./xmlchange PTS_LAT=74.5 +./xmlchange PTS_LON=10.0 + +./xmlchange SSTICE_DATA_FILENAME="${DIN_LOC_ROOT}/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc" +./xmlchange SSTICE_YEAR_ALIGN=2020 +./xmlchange SSTICE_YEAR_START=2020 +./xmlchange SSTICE_YEAR_END=2021 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/COMBLE_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=74.5 -b +$ATMCHANGE target_longitude=10.0 -b +$ATMCHANGE iop_coriolis=true + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands index 58c235807872..8c442acee75b 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands @@ -8,7 +8,7 @@ # Scripts location ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange -$ATMCHANGE iop_file=${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -b +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -b $ATMCHANGE target_latitude=31.5 -b $ATMCHANGE target_longitude=238.5 -b $ATMCHANGE iop_dosubsidence=true -b From 29d83eba554da13a41dc590ea4354c2585d1686d Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 30 Jul 2024 11:15:57 -0600 Subject: [PATCH 224/477] Revert "Changes due to code review feedback." This reverts commit 8fd0eae1ad3d3ae389b10e1e685d4061714e256d. There are fields that need to be in the FieldManager since this is still in the evaluation stage. --- .../eamxx_mam_wetscav_process_interface.cpp | 38 +++++++++++++------ .../eamxx_mam_wetscav_process_interface.hpp | 12 +----- .../mam/p3_mam4_wetscav/output.yaml | 4 ++ .../mam/shoc_cldfrac_p3_wetscav/output.yaml | 4 ++ .../single-process/mam/wet_scav/output.yaml | 4 ++ 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 81f13fd6ae9a..2a81c1a9c427 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -47,6 +47,10 @@ void MAMWetscav::set_grids( // layout for 2D (1d horiz X 1d vertical) variables FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + // layout for 3D (ncol, nmodes, nlevs) + FieldLayout scalar3d_mid_nmodes = + m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); + // layout for 2D (ncol, pcnst) FieldLayout scalar2d_pconst = m_grid->get_2d_vector_layout(pcnst, "num_phys_constants"); @@ -201,6 +205,22 @@ void MAMWetscav::set_grids( // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- + static constexpr auto m3 = m * m * m; + + // Aerosol dry particle diameter [m] + add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); + + // Wet aerosol density [kg/m3] + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); + + // Aerosol water [kg/kg] + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); + + // Wet aerosol diameter [m] + add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); + + // Fraction of transported species that are insoluble [fraction] + add_field("fracis", scalar3d_mid, nondim, grid_name); // Aerosol wet deposition (interstitial) [kg/m2/s] add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); @@ -332,14 +352,6 @@ void MAMWetscav::initialize_impl(const RunType run_type) { } } - const int nmodes = mam4::AeroConfig::num_modes(); - - // Aerosol dry particle diameter [m] - dgncur_a_ = view_3d("dgncur_a", ncol_, nmodes, nlev_); - wetdens_ = view_3d("wetdens", ncol_, nmodes, nlev_); - qaerwat_ = view_3d("qaerwat", ncol_, nmodes, nlev_); - dgnumwet_ = view_3d("dgnumwet", ncol_, nmodes, nlev_); - // Allocate work array const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); work_ = view_2d("work", ncol_, work_len); @@ -463,10 +475,12 @@ void MAMWetscav::run_impl(const double dt) { const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); - const auto wet_geometric_mean_diameter_i = dgnumwet_; - const auto dry_geometric_mean_diameter_i = dgncur_a_; - const auto qaerwat = qaerwat_; - const auto wetdens = wetdens_; + const auto wet_geometric_mean_diameter_i = + get_field_out("dgnumwet").get_view(); + const auto dry_geometric_mean_diameter_i = + get_field_out("dgncur_a").get_view(); + const auto qaerwat = get_field_out("qaerwat").get_view(); + const auto wetdens = get_field_out("wetdens").get_view(); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp index 3c3dec6cc623..9d1dc20bc88f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -23,7 +23,6 @@ namespace scream { class MAMWetscav : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; using view_2d = typename KT::template view_2d; - using view_3d = typename KT::template view_3d; // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; @@ -161,18 +160,9 @@ class MAMWetscav : public scream::AtmosphereProcess { // Work arrays view_2d work_; - // Aerosol dry particle diameter [m] - // Dimensions: [cols, modes, levels] - view_3d dgncur_a_; - // Wet aerosol density [kg/m3] - view_3d wetdens_; - // Aerosol water [kg/kg] - view_3d qaerwat_; - // Wet aerosol diameter [m] - view_3d dgnumwet_; - // TODO: Following variables are from convective parameterization (not // implemented yet in EAMxx), so should be zero for now + view_2d sh_frac_; // Deep convective cloud fraction [fraction] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml index 049dcdad35a5..c26db8eb6814 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -56,6 +56,10 @@ Field Names: - num_c4 - aerdepwetis - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml index c38bafaa65e3..1ae53e6c1c30 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -90,6 +90,10 @@ Field Names: - num_c4 - aerdepwetis - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml index 75172823870e..9cd0a16bc414 100644 --- a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -7,6 +7,10 @@ Fields: Field Names: - aerdepwetis - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat - bc_c1 - bc_c3 - bc_c4 From fc80af369da2416e71d5b5f68196c7cb3c1ef023 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 30 Jul 2024 11:03:40 -0700 Subject: [PATCH 225/477] update comment --- components/eamxx/cime_config/config_compsets.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index c41bdb870ec7..c684e1f6c6b6 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -75,7 +75,7 @@ - F2010-SCREAMv1-DP-General + F2010-SCREAMv1-DP-General 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%General Experimental, under development From ffe22ad4c463e6fce1c8f9fbec2daab1089fd6d3 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 11 Jul 2024 12:45:16 -0700 Subject: [PATCH 226/477] read in IOP lev data before surface pressure adjustment is applied --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 657f7ad742a0..70f1c41b603a 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -541,6 +541,15 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); + // Read in IOP lev data + auto data = iop_file_pressure.get_view().data(); + scorpio::read_var(iop_file,"lev",data); + + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilev(); From 3a6fcc9c5500030580d6eac5ec6716748e978128 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 11 Jul 2024 16:21:55 -0700 Subject: [PATCH 227/477] add minimum threshold when checking lat and lon relative error for IOP file to prevent NaN --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 70f1c41b603a..42ea7d0ec03f 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - m_params.get("target_latitude"); + ekat::impl::max(m_params.get("target_latitude"),0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - m_params.get("target_longitude"); + ekat::impl::max(m_params.get("target_longitude"),0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), From 613beffa39b50655cfb28430d68d91bff5ed52b1 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 16 Jul 2024 10:28:54 -0700 Subject: [PATCH 228/477] change ekat::impl::max instances to std::max and remove unneeded m_helper_fields line --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 42ea7d0ec03f..cf20435fd2ce 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - ekat::impl::max(m_params.get("target_latitude"),0.1); + std::max(m_params.get("target_latitude"),0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - ekat::impl::max(m_params.get("target_longitude"),0.1); + std::max(m_params.get("target_longitude"),0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -548,7 +548,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) // Convert to pressure to millibar (file gives pressure in Pa) for (int ilev=0; ilev Date: Wed, 17 Jul 2024 09:42:48 -0700 Subject: [PATCH 229/477] cast minimum lat/lon threshold as a real --- .../eamxx/src/share/iop/intensive_observation_period.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index cf20435fd2ce..6f8474f8d85e 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -268,9 +268,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),0.1); + std::max(m_params.get("target_latitude"),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - std::max(m_params.get("target_longitude"),0.1); + std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), From 861565c882d829a70e9375a3c800b34ea67af165 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 19 Jul 2024 15:04:44 -0400 Subject: [PATCH 230/477] fix capture this warning in mam micro --- .../physics/mam/eamxx_mam_microphysics_process_interface.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7ea2adfddb63..8b93936148ce 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,6 +317,8 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index @@ -448,7 +450,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, some_step, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, From 44be05960805fc3539a2736ecef3072c3dfeaeab Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:10:21 -0500 Subject: [PATCH 231/477] Add ums and ums002 to load libunwind 1.6.2 After recent Frontier maintenance, ums/default and ums002/default are required to load libunwind/1.6.2. --- cime_config/machines/config_machines.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index c04dff5976cd..3617bb88f9ec 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1437,6 +1437,8 @@ PrgEnv-cray craype-accel-amd-gfx90a rocm/5.4.0 + ums/default + ums002/default libunwind/1.6.2 From 08fb2d1bd4f14ba8a1e279b049129f61b86546f3 Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:12:27 -0500 Subject: [PATCH 232/477] Add to scream_scorpio_types.hpp to use std::int64_t This fixes a known build error after recent Frontier maintenance: no type named 'int64_t' in namespace 'std'; did you mean simply 'int64_t'? --- components/eamxx/src/share/io/scream_scorpio_types.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/share/io/scream_scorpio_types.hpp b/components/eamxx/src/share/io/scream_scorpio_types.hpp index 645b1b26937b..3b16cebc06c8 100644 --- a/components/eamxx/src/share/io/scream_scorpio_types.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_types.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace scream { namespace scorpio { From ac7280cdaea785027609f1306798a44ae806294e Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 19 Jul 2024 17:16:28 -0500 Subject: [PATCH 233/477] Explicitly add gcc/12.2.0 lib path to CMAKE_EXE_LINKER_FLAGS This is a simple workaround to fix scream linking errors after recent Frontier maintenance before we find better solutions. --- .../cmake_macros/crayclang-scream_frontier-scream-gpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index afcca8f479e5..cc90b369244a 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -19,7 +19,7 @@ endif() string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64 -L/opt/gcc/12.2.0/snos/lib64") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{ROCM_PATH}/include") # Crusher: this resolves a crash in mct in docn init From bcf0a02e6e053b67eea98d8eb0dbec03b3f2dbbd Mon Sep 17 00:00:00 2001 From: Hassan Beydoun Date: Fri, 19 Jul 2024 13:05:12 -0700 Subject: [PATCH 234/477] add output variable for rain effective diameter --- components/eam/src/physics/p3/scream/micro_p3_interface.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 index edea27959d25..11d13c71a7a7 100644 --- a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 @@ -834,6 +834,8 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) real(rtype) :: icwmrst(pcols,pver) ! stratus water mixing ratio - on grid real(rtype) :: rho(pcols,pver) real(rtype) :: reff_rain(pcols,pver) + real(rtype) :: deff_rain(pcols,pver) + real(rtype) :: col_location(pcols,3),tmp_loc(pcols) ! Array of column lon (index 1) and lat (index 2) integer :: tmpi_loc(pcols) ! Global column index temp array @@ -1358,6 +1360,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) aqrain = 0._rtype anrain = 0._rtype freqr = 0._rtype + deff_rain = reff_rain*2._rtype*1.e6_rtype !rain effective diameter in microns ! Prognostic precipitation where (rain(:ncol,top_lev:) >= 1.e-7_rtype) aqrain(:ncol,top_lev:) = rain(:ncol,top_lev:) * cld_frac_r(:ncol,top_lev:) @@ -1412,6 +1415,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) call outfld('FREQI', freqi, pcols, lchnk) call outfld('FREQR', freqr, pcols, lchnk) call outfld('CDNUMC', cdnumc, pcols, lchnk) + call outfld('ADRAIN', deff_rain, pcols, lchnk) call outfld('CLOUDFRAC_LIQ_MICRO', cld_frac_l, pcols, lchnk) call outfld('CLOUDFRAC_ICE_MICRO', cld_frac_i, pcols, lchnk) From e55a1217ed8723f0623b8397a07dfb00522c1b27 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 18 Jul 2024 15:46:35 -0600 Subject: [PATCH 235/477] Add ustar and obklen to FM --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 657f9487611c..80485bf67130 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -395,6 +395,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); + add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. From 4d4fc7492c359f1094a3bed5b5a027c986dc1333 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 08:43:37 -0600 Subject: [PATCH 236/477] Remove obklen postcondition check --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 80485bf67130..657f9487611c 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -395,7 +395,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) add_postcondition_check(get_field_out("qc"),m_grid,0.0,0.1,false); add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); add_postcondition_check(get_field_out("pbl_height"),m_grid,0); - add_postcondition_check(get_field_out("obklen"),m_grid,0); add_postcondition_check(get_field_out("cldfrac_liq"),m_grid,0.0,1.0,false); add_postcondition_check(get_field_out("tke"),m_grid,0); // For qv, ensure it doesn't get negative, by allowing repair of any neg value. From 237c7f524f01291055a4b3dff72c891c8072e328 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 22 Jul 2024 10:20:20 -0600 Subject: [PATCH 237/477] Fix how IOP reads time --- .../src/share/iop/intensive_observation_period.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 6f8474f8d85e..32b120c57002 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -248,15 +248,17 @@ initialize_iop_file(const util::TimeStamp& run_t0, else if (scorpio::has_dim(iop_file, "tsec")) time_dimname = "tsec"; else EKAT_ERROR_MSG("Error! No valid dimension for tsec in "+iop_file+".\n"); - const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); - m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); - scorpio::read_var(iop_file,"tsec",m_time_info.iop_file_times_in_sec.data()); - - // From now on, when we read vars, "time" must be treated as unlimited, to avoid issues + // When we read vars, "time" must be treated as unlimited, to avoid issues if (not scorpio::is_dim_unlimited(iop_file,time_dimname)) { scorpio::pretend_dim_is_unlimited(iop_file,time_dimname); } + const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); + m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); + for (int t=0; t Date: Fri, 19 Jul 2024 15:48:20 -0700 Subject: [PATCH 238/477] filter aodvis by day indices --- components/eamxx/src/diagnostics/aodvis.cpp | 19 ++++++-- .../src/diagnostics/tests/aodvis_test.cpp | 46 ++++++++++++++++--- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index efb9b5ba9510..ac16c0033730 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -2,6 +2,8 @@ #include +#include "share/util/scream_universal_constants.hpp" + namespace scream { AODVis::AODVis(const ekat::Comm &comm, const ekat::ParameterList ¶ms) @@ -23,11 +25,13 @@ void AODVis::set_grids( m_nlevs = grid->get_num_vertical_levels(); // Define layouts we need (both inputs and outputs) - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,m_swbands,"swband"); - FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, m_swbands, "swband"); + FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); // The fields required for this diagnostic to be computed add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + add_field("sunlit", scalar1d_layout, nondim, grid_name); // Construct and allocate the aodvis field FieldIdentifier fid("AerosolOpticalDepth550nm", scalar1d_layout, nondim, @@ -41,18 +45,25 @@ void AODVis::compute_diagnostic_impl() { using MT = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; + Real var_fill_value = constants::DefaultFillValue().value; + const auto aod = m_diagnostic_output.get_view(); const auto tau_vis = get_field_in("aero_tau_sw") .subfield(1, m_vis_bnd) .get_view(); + const auto sunlit = get_field_in("sunlit").get_view(); const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); - auto tau_icol = ekat::subview(tau_vis, icol); - aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + if(sunlit(icol) == 0.0) { + aod(icol) = var_fill_value; + } else { + auto tau_icol = ekat::subview(tau_vis, icol); + aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + } }); } diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 140114812215..15de175da67d 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -1,10 +1,11 @@ +#include + #include "catch2/catch.hpp" #include "diagnostics/register_diagnostics.hpp" #include "share/field/field_utils.hpp" #include "share/grid/mesh_free_grids_manager.hpp" #include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_utils.hpp" - +#include "share/util/scream_universal_constants.hpp" namespace scream { std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, @@ -30,6 +31,10 @@ TEST_CASE("aodvis") { using namespace ShortFieldTagsNames; using namespace ekat::units; + Real var_fill_value = constants::DefaultFillValue().value; + + Real some_limit = 0.0025; + // A world comm ekat::Comm comm(MPI_COMM_WORLD); @@ -40,7 +45,7 @@ TEST_CASE("aodvis") { // Create a grids manager - single column for these tests constexpr int nlevs = 33; - const int ngcols = 2 * comm.size(); + const int ngcols = 10 * comm.size(); int nbnds = eamxx_swbands(); int swvis = eamxx_vis_swband_idx(); @@ -49,16 +54,23 @@ TEST_CASE("aodvis") { auto grid = gm->get_grid("Physics"); // Input (randomized) tau - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,nbnds,"swband"); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, nbnds, "swband"); FieldIdentifier tau_fid("aero_tau_sw", scalar3d_swband_layout, nondim, grid->name()); Field tau(tau_fid); tau.allocate_view(); tau.get_header().get_tracking().update_time_stamp(t0); + // Input (randomized) sunlit + FieldLayout scalar2d_layout = grid->get_2d_scalar_layout(); + FieldIdentifier sunlit_fid("sunlit", scalar2d_layout, nondim, grid->name()); + Field sunlit(sunlit_fid); + sunlit.allocate_view(); + sunlit.get_header().get_tracking().update_time_stamp(t0); // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(0, 0.05); + RPDF pdf(0, 0.005); auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -71,18 +83,33 @@ TEST_CASE("aodvis") { // Randomize tau randomize(tau, engine, pdf); + // Randomize sunlit + randomize(sunlit, engine, pdf); + // Create and set up the diagnostic ekat::ParameterList params; auto diag = diag_factory.create("AerosolOpticalDepth550nm", comm, params); diag->set_grids(gm); diag->set_required_field(tau); + diag->set_required_field(sunlit); diag->initialize(t0, RunType::Initial); + auto sun_h = sunlit.get_view(); + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + // zero out all sun_h if below 0.05 + if(sun_h(icol) < some_limit) { + sun_h(icol) = 0.0; + } + } + // sync to device + sunlit.sync_to_dev(); + // Run diag diag->compute_diagnostic(); // Check result tau.sync_to_host(); + sunlit.sync_to_host(); diag->get_diagnostic().sync_to_host(); const auto tau_h = tau.get_view(); @@ -93,12 +120,17 @@ TEST_CASE("aodvis") { auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - for(int ilev = 0; ilev < nlevs; ++ilev) { - aod_t(icol) += tau_h(icol, swvis, ilev); + if(sun_h(icol) < some_limit) { + aod_t(icol) = var_fill_value; + } else { + for(int ilev = 0; ilev < nlevs; ++ilev) { + aod_t(icol) += tau_h(icol, swvis, ilev); + } } } aod_hf.sync_to_dev(); aod_tf.sync_to_dev(); + // Workaround for non-bfb behavior of view_reduction() in release builds if(SCREAM_BFB_TESTING) { REQUIRE(views_are_equal(aod_hf, aod_tf)); From 7d8d166ec3d48d8465c982b85f277fa6c6ccf957 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 25 Jul 2024 22:24:07 -0400 Subject: [PATCH 239/477] remove sunlit sync to host --- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 15de175da67d..8090a9300fd9 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -109,7 +109,6 @@ TEST_CASE("aodvis") { // Check result tau.sync_to_host(); - sunlit.sync_to_host(); diag->get_diagnostic().sync_to_host(); const auto tau_h = tau.get_view(); From 5230b6081fb88817921b49460ee5b51d92e65e2b Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 12 Jul 2024 16:42:47 -0500 Subject: [PATCH 240/477] rename pyeamxx to pyscream --- components/eamxx/src/python/CMakeLists.txt | 4 ++-- components/eamxx/src/python/pyatmproc.hpp | 2 +- components/eamxx/src/python/pyeamxx.cpp | 2 +- components/eamxx/src/python/pyeamxx.hpp | 6 +++--- components/eamxx/src/python/pyeamxx/__init__.py | 16 ++++++++-------- components/eamxx/src/python/pygrid.hpp | 4 ++-- components/eamxx/src/python/pyproject.toml | 6 +++--- components/eamxx/src/python/readme | 6 +++--- .../eamxx/tests/python/pyp3/CMakeLists.txt | 2 +- .../eamxx/tests/python/pyp3/p3_standalone_py | 12 ++++++------ 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 43e2db73e533..41eae6d65463 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,8 +1,8 @@ find_package(pybind11 REQUIRED) find_package(mpi4py REQUIRED) -pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC +pybind11_add_module(pyscream pyscream.cpp) +target_link_libraries(pyscream PUBLIC mpi4py scream_share scream_io diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index 98d7b9a525ed..9e45bc513f1b 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -8,7 +8,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" -#include "pyeamxx.hpp" +#include "pyscream.hpp" #include diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index c4ec360d80a1..2efb8b6f8df9 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -45,7 +45,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyeamxx,m) { +PYBIND11_MODULE (pyscream,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/pyeamxx.hpp index 59371844a763..781c195919dd 100644 --- a/components/eamxx/src/python/pyeamxx.hpp +++ b/components/eamxx/src/python/pyeamxx.hpp @@ -1,5 +1,5 @@ -#ifndef PYEAMXX_HPP -#define PYEAMXX_HPP +#ifndef PYSCREAM_HPP +#define PYSCREAM_HPP #include "physics/register_physics.hpp" #include "diagnostics/register_diagnostics.hpp" @@ -27,4 +27,4 @@ struct PySession { } // namespace scream -#endif // PYEAMXX_HPP +#endif // PYSCREAM_HPP diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py index 75a36e148291..98c31744eb9f 100644 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ b/components/eamxx/src/python/pyeamxx/__init__.py @@ -1,15 +1,15 @@ """ This file will serve as a way to organize and expose - libpyeamxx internals to the rest of pyeamxx + libpyscream internals to the rest of pyscream """ -from libpyeamxx.libpyeamxx_ext import AtmProc -from libpyeamxx.libpyeamxx_ext import Grid -from libpyeamxx.libpyeamxx_ext import ParameterList -from libpyeamxx.libpyeamxx_ext import init -from libpyeamxx.libpyeamxx_ext import Field -from libpyeamxx.libpyeamxx_ext import P3 -from libpyeamxx.libpyeamxx_ext import finalize +from libpyscream.libpyscream_ext import AtmProc +from libpyscream.libpyscream_ext import Grid +from libpyscream.libpyscream_ext import ParameterList +from libpyscream.libpyscream_ext import init +from libpyscream.libpyscream_ext import Field +from libpyscream.libpyscream_ext import P3 +from libpyscream.libpyscream_ext import finalize __all__ = [ diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp index 102cf6037c62..e49a0443a345 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/pygrid.hpp @@ -3,7 +3,7 @@ #include "share/grid/mesh_free_grids_manager.hpp" -#include "pyeamxx.hpp" +#include "pyscream.hpp" #include @@ -14,7 +14,7 @@ namespace scream { inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) { EKAT_REQUIRE_MSG (PySession::get().inited, - "Error! You did not initialize pyeamxx, or you already finalized it!\n"); + "Error! You did not initialize pyscream, or you already finalized it!\n"); auto& comm = PySession::get().comm; ekat::ParameterList gm_params; std::vector grids_names = {"Physics"}; diff --git a/components/eamxx/src/python/pyproject.toml b/components/eamxx/src/python/pyproject.toml index fa5d6fc44817..490ea99c143e 100644 --- a/components/eamxx/src/python/pyproject.toml +++ b/components/eamxx/src/python/pyproject.toml @@ -3,15 +3,15 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "pyeamxx" +name = "pyscream" version = "0.0.2" dependencies = ["numpy", "mpi4py"] [tool.setuptools.packages.find] where = ["."] -include = ["pyeamxx", "libpyeamxx"] +include = ["pyscream", "libpyscream"] exclude = ["build_src", "tests"] namespaces = true [tool.setuptools.package-data] -"libpyeamxx" = ["*.so*"] +"libpyscream" = ["*.so*"] diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme index f5ed48f058ee..c16c32a5d217 100644 --- a/components/eamxx/src/python/readme +++ b/components/eamxx/src/python/readme @@ -1,7 +1,7 @@ INFO: - EAMxx python bindings -- pyeamxx is where we will house the python code -- libpyeamxx is where we will house the extensions +- pyscream is where we will house the python code +- libpyscream is where we will house the extensions - packaging moved to https://github.com/mahf708/experimental-scream-feedstock TODO: @@ -11,5 +11,5 @@ TODO: - decide archs/pythons/mpis to tgt USER: -- conda install pyeamxx -c mahf708/label/$mac (mac is chrysalis or pm-cpu) +- conda install pyscream -c mahf708/label/$mac (mac is chrysalis or pm-cpu) - see example in components/eamxx/tests/python/pyp3 diff --git a/components/eamxx/tests/python/pyp3/CMakeLists.txt b/components/eamxx/tests/python/pyp3/CMakeLists.txt index 7142be8dd8f8..945f15faaf65 100644 --- a/components/eamxx/tests/python/pyp3/CMakeLists.txt +++ b/components/eamxx/tests/python/pyp3/CMakeLists.txt @@ -53,6 +53,6 @@ foreach (rank IN LISTS MpiRanks) SRC_FILE p3_standalone_cxx.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc TGT_FILE p3_standalone_py.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_cxx${suffix} ${FIXTURES_BASE_NAME}_py${suffix} - LABELS pyeamxx + LABELS pyscream ) endforeach() diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index 0db6bcfcef61..34747943d081 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -5,7 +5,7 @@ import sys # Add path to scream libs sys.path.append('@SCREAM_BASE_DIR@/scripts') -# Add path to pyeamxx libs +# Add path to pyscream libs sys.path.append('@CMAKE_BINARY_DIR@/src/python') # Without these, and manual init/finalize, on my laptop I get @@ -15,7 +15,7 @@ mpi4py.rc.initialize = False # do not initialize MPI automatically mpi4py.rc.finalize = False # do not finalize MPI automatically from mpi4py import MPI -import pyeamxx +import pyscream from pathlib import Path from utils import ensure_yaml @@ -38,9 +38,9 @@ def main (): # Create the grid ncols = 218 nlevs = 72 - pyeamxx.create_grids_manager(ncols,nlevs,str(ic_file)) + pyscream.create_grids_manager(ncols,nlevs,str(ic_file)) - p3 = pyeamxx.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') + p3 = pyscream.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') params = p3.get_params() old = params.get_dbl('max_total_ni') print (f"max_total_ni: {params.get_dbl('max_total_ni')}") @@ -64,7 +64,7 @@ if __name__ == "__main__": # This level of indirection ensures all pybind structs are destroyed # before we finalize eamxx (and hence kokkos) MPI.Init() - pyeamxx.init() + pyscream.init() main () - pyeamxx.finalize() + pyscream.finalize() MPI.Finalize() From a6f14a9d4482df92de08ac5eb8e56639ddd02a38 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 12 Jul 2024 16:54:10 -0500 Subject: [PATCH 241/477] more renaming --- components/eamxx/src/python/CMakeLists.txt | 13 +---------- .../src/python/libpyscream/CMakeLists.txt | 12 ++++++++++ .../python/{ => libpyscream}/pyatmproc.hpp | 2 +- .../src/python/{ => libpyscream}/pyfield.hpp | 0 .../src/python/{ => libpyscream}/pygrid.hpp | 2 +- .../python/{ => libpyscream}/pyparamlist.hpp | 0 .../pyscream_ext.cpp} | 2 +- .../pyscream_ext.hpp} | 0 .../src/python/{ => libpyscream}/pyutils.hpp | 0 .../eamxx/src/python/pyeamxx/__init__.py | 23 ------------------- .../eamxx/src/python/pyscream/__init__.py | 10 ++++++++ 11 files changed, 26 insertions(+), 38 deletions(-) create mode 100644 components/eamxx/src/python/libpyscream/CMakeLists.txt rename components/eamxx/src/python/{ => libpyscream}/pyatmproc.hpp (99%) rename components/eamxx/src/python/{ => libpyscream}/pyfield.hpp (100%) rename components/eamxx/src/python/{ => libpyscream}/pygrid.hpp (97%) rename components/eamxx/src/python/{ => libpyscream}/pyparamlist.hpp (100%) rename components/eamxx/src/python/{pyeamxx.cpp => libpyscream/pyscream_ext.cpp} (97%) rename components/eamxx/src/python/{pyeamxx.hpp => libpyscream/pyscream_ext.hpp} (100%) rename components/eamxx/src/python/{ => libpyscream}/pyutils.hpp (100%) delete mode 100644 components/eamxx/src/python/pyeamxx/__init__.py create mode 100644 components/eamxx/src/python/pyscream/__init__.py diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 41eae6d65463..4c8adf40fca7 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,12 +1 @@ -find_package(pybind11 REQUIRED) -find_package(mpi4py REQUIRED) - -pybind11_add_module(pyscream pyscream.cpp) -target_link_libraries(pyscream PUBLIC - mpi4py - scream_share - scream_io - diagnostics - eamxx_physics - scream_test_support -) +add_subdirectory(libpyscream) \ No newline at end of file diff --git a/components/eamxx/src/python/libpyscream/CMakeLists.txt b/components/eamxx/src/python/libpyscream/CMakeLists.txt new file mode 100644 index 000000000000..103c6a80b888 --- /dev/null +++ b/components/eamxx/src/python/libpyscream/CMakeLists.txt @@ -0,0 +1,12 @@ +find_package(pybind11 REQUIRED) +find_package(mpi4py REQUIRED) + +pybind11_add_module(pyscream_ext pyscream_ext.cpp) +target_link_libraries(pyscream_ext PUBLIC + mpi4py + scream_share + scream_io + diagnostics + eamxx_physics + scream_test_support +) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp similarity index 99% rename from components/eamxx/src/python/pyatmproc.hpp rename to components/eamxx/src/python/libpyscream/pyatmproc.hpp index 9e45bc513f1b..0acc371c1f71 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -8,7 +8,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" -#include "pyscream.hpp" +#include "pyscream_ext.hpp" #include diff --git a/components/eamxx/src/python/pyfield.hpp b/components/eamxx/src/python/libpyscream/pyfield.hpp similarity index 100% rename from components/eamxx/src/python/pyfield.hpp rename to components/eamxx/src/python/libpyscream/pyfield.hpp diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp similarity index 97% rename from components/eamxx/src/python/pygrid.hpp rename to components/eamxx/src/python/libpyscream/pygrid.hpp index e49a0443a345..e8adfa3bbc49 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -3,7 +3,7 @@ #include "share/grid/mesh_free_grids_manager.hpp" -#include "pyscream.hpp" +#include "pyscream_ext.hpp" #include diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/libpyscream/pyparamlist.hpp similarity index 100% rename from components/eamxx/src/python/pyparamlist.hpp rename to components/eamxx/src/python/libpyscream/pyparamlist.hpp diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp similarity index 97% rename from components/eamxx/src/python/pyeamxx.cpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.cpp index 2efb8b6f8df9..e0ad9ceec260 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp @@ -45,7 +45,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyscream,m) { +PYBIND11_MODULE (pyscream_ext,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx.hpp b/components/eamxx/src/python/libpyscream/pyscream_ext.hpp similarity index 100% rename from components/eamxx/src/python/pyeamxx.hpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.hpp diff --git a/components/eamxx/src/python/pyutils.hpp b/components/eamxx/src/python/libpyscream/pyutils.hpp similarity index 100% rename from components/eamxx/src/python/pyutils.hpp rename to components/eamxx/src/python/libpyscream/pyutils.hpp diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py deleted file mode 100644 index 98c31744eb9f..000000000000 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" - This file will serve as a way to organize and expose - libpyscream internals to the rest of pyscream -""" - -from libpyscream.libpyscream_ext import AtmProc -from libpyscream.libpyscream_ext import Grid -from libpyscream.libpyscream_ext import ParameterList -from libpyscream.libpyscream_ext import init -from libpyscream.libpyscream_ext import Field -from libpyscream.libpyscream_ext import P3 -from libpyscream.libpyscream_ext import finalize - - -__all__ = [ - 'init', - 'finalize', - 'AtmProc', - 'Grid', - 'ParameterList', - 'Field', - 'P3', -] diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py new file mode 100644 index 000000000000..8159f5da47c7 --- /dev/null +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -0,0 +1,10 @@ +""" + This file will serve as a way to organize and expose + libpyscream internals to the rest of pyscream +""" + +from libpyscream import pyscream_ext as pyscream + +__all__ = [ + "pyscream" +] From 8c81c698d7a1a6b5c61292884f2f64acd185b464 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 12:06:01 -0500 Subject: [PATCH 242/477] skip cosp when building python --- components/eamxx/src/physics/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e0e89e60f80d..e5bce027bcc4 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -7,7 +7,11 @@ add_subdirectory(share) add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) - add_subdirectory(cosp) + if (EAMXX_ENABLE_PYBIND) + message(STATUS "WARNING: skipping COSP when buidling Python") + else() + add_subdirectory(cosp) + endif() else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() From ae7dfedd0778f743e1ab0df24d15d12b4211ffeb Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 14:02:39 -0500 Subject: [PATCH 243/477] Add geo_data_source --- components/eamxx/src/python/libpyscream/pygrid.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index e8adfa3bbc49..93a8a5395dba 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -25,6 +25,7 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo gm_params.set("grids_names",grids_names); if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } From a38380b9cf2928404ba618a1d91e29f2978a57a0 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 13 Jul 2024 23:04:05 -0500 Subject: [PATCH 244/477] dump fields and reorg things --- components/eamxx/src/physics/CMakeLists.txt | 6 +----- components/eamxx/src/python/CMakeLists.txt | 2 +- .../eamxx/src/python/libpyscream/pyatmproc.hpp | 9 +++++++++ components/eamxx/src/python/libpyscream/pygrid.hpp | 1 - components/eamxx/src/python/pyscream/__init__.py | 14 ++++++++++++-- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e5bce027bcc4..e0e89e60f80d 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -7,11 +7,7 @@ add_subdirectory(share) add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) - if (EAMXX_ENABLE_PYBIND) - message(STATUS "WARNING: skipping COSP when buidling Python") - else() - add_subdirectory(cosp) - endif() + add_subdirectory(cosp) else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 4c8adf40fca7..d19bc05c240e 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1 +1 @@ -add_subdirectory(libpyscream) \ No newline at end of file +add_subdirectory(libpyscream) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 0acc371c1f71..5248ccf1f965 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -131,6 +131,14 @@ struct PyAtmProc { return pybind11::cast(missing); } + pybind11::list dump_fields() { + std::vector all_fields; + for (auto it : fields) { + all_fields.push_back(it.first); + } + return pybind11::cast(all_fields); + } + void setup_output (const std::string& yaml_file) { auto comm = PySession::get().comm; @@ -176,6 +184,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) .def("read_ic",&PyAtmProc::read_ic); + .def("dump_fields",&PyAtmProc::dump_fields); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index 93a8a5395dba..e8adfa3bbc49 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -25,7 +25,6 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo gm_params.set("grids_names",grids_names); if (latlon_nc_file!="") { - gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py index 8159f5da47c7..e76f52170fb3 100644 --- a/components/eamxx/src/python/pyscream/__init__.py +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -3,8 +3,18 @@ libpyscream internals to the rest of pyscream """ -from libpyscream import pyscream_ext as pyscream +from libpyscream.pyscream_ext import init +from libpyscream.pyscream_ext import finalize +from libpyscream.pyscream_ext import Field +from libpyscream.pyscream_ext import AtmProc +from libpyscream.pyscream_ext import ParameterList +from libpyscream.pyscream_ext import create_grids_manager __all__ = [ - "pyscream" + "init", + "finalize", + "Field", + "AtmProc", + "ParameterList", + "create_grids_manager", ] From 2c8d6a7a3757742d56bc85248f9add846b019ca4 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 14 Jul 2024 09:56:44 -0500 Subject: [PATCH 245/477] fix minor typo in def dumpfields --- components/eamxx/src/python/libpyscream/pyatmproc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 5248ccf1f965..000a7d64c9a9 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -183,7 +183,7 @@ inline void pybind_pyatmproc(pybind11::module& m) .def("get_params",&PyAtmProc::get_params) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) - .def("read_ic",&PyAtmProc::read_ic); + .def("read_ic",&PyAtmProc::read_ic) .def("dump_fields",&PyAtmProc::dump_fields); } } // namespace scream From fc7b9e6a2961da698f2d54059feadfdfc568fedb Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 14 Jul 2024 11:48:48 -0500 Subject: [PATCH 246/477] add geo data source --- components/eamxx/src/python/libpyscream/pygrid.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index e8adfa3bbc49..1c0d56eb08b3 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -23,8 +23,10 @@ inline void create_grids_manager (int ncols, int nlevs, const std::string& latlo pl.set("number_of_global_columns",ncols); pl.set("number_of_vertical_levels",nlevs); gm_params.set("grids_names",grids_names); + gm_params.set("geo_data_source",std::string("CREATE_EMPTY_DATA")); if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); gm_params.set("ic_filename",latlon_nc_file); } From c1344ad585738915bbb9a2cf076d3bdb0f1776a3 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 5 Jun 2024 17:08:33 -0600 Subject: [PATCH 247/477] Port scream_rrtmgp_interface to new pattern --- .../eamxx/src/physics/rrtmgp/CMakeLists.txt | 1 - .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 6 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 1167 +------------- .../rrtmgp/scream_rrtmgp_interface.hpp | 1432 ++++++++++++++--- 4 files changed, 1243 insertions(+), 1363 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index fcb819f599a8..0d42d61fa662 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -109,7 +109,6 @@ set(EAM_RRTMGP_DIR ${SCREAM_BASE_DIR}/../eam/src/physics/rrtmgp) # NOTE: The external RRTMGP build needs some fixes to work with CUDA in a library build, so for now we will build these ourselves set(EXTERNAL_SRC ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels/mo_gas_optics_kernels.cpp - ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_constants.cpp ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_util_reorder.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/expand_and_transpose.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_fluxes_broadband_kernels.cpp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 0dfe4945e0c8..119093d5f9d3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -109,14 +109,14 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, RealT xmin, RealT xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - bool1dk bad_mask("bad_mask", x.size()); + Kokkos::View bad_mask("bad_mask", x.size()); Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { if (x.data()[i] < xmin or x.data()[i] > xmax) { bad_mask.data()[i] = true; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 450d7ad216be..1696e7933685 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,5 +1,4 @@ #include "scream_rrtmgp_interface.hpp" -#include "rrtmgp_utils.hpp" #include "cpp/examples/mo_load_coefficients.h" #include "examples/all-sky/mo_load_cloud_coefficients.h" #include "cpp/rrtmgp/mo_gas_concentrations.h" @@ -8,10 +7,6 @@ #include "cpp/rte/mo_rte_sw.h" #include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" -#include "ekat/util/ekat_math_utils.hpp" -#ifdef RRTMGP_ENABLE_KOKKOS -#include "Kokkos_Random.hpp" -#endif namespace scream { @@ -38,41 +33,27 @@ void finalize_kls() #endif } +#ifdef RRTMGP_ENABLE_YAKL namespace rrtmgp { -#ifdef RRTMGP_ENABLE_YAKL using yakl::fortran::parallel_for; using yakl::fortran::SimpleBounds; using yakl::intrinsics::merge; -#endif - /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -#ifdef RRTMGP_ENABLE_YAKL GasOpticsRRTMGP k_dist_sw; GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -GasOpticsRRTMGPK k_dist_sw_k; -GasOpticsRRTMGPK k_dist_lw_k; -#endif /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -#ifdef RRTMGP_ENABLE_YAKL CloudOptics cloud_optics_sw; CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -CloudOpticsK cloud_optics_sw_k; -CloudOpticsK cloud_optics_lw_k; -#endif bool initialized = false; bool initialized_k = false; @@ -80,7 +61,6 @@ bool initialized_k = false; // local functions namespace { -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_cloud_optics_sw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -106,36 +86,7 @@ OpticalProps2str get_cloud_optics_sw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_cloud_optics_sw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps2strK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_2str(ncol, nlay); - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_cloud_optics_lw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -161,36 +112,7 @@ OpticalProps1scl get_cloud_optics_lw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_cloud_optics_lw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps1sclK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2str &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -242,60 +164,7 @@ OpticalProps2str get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps2strK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_2str(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, - // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), - // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should - // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very - // high resolution. - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); - subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - subsampled_optics.ssa(icol,ilay,igpt) = 0; - subsampled_optics.g (icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1scl &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -339,51 +208,6 @@ OpticalProps1scl get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_1scl(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM; use different - // seed values for longwave and shortwave - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif } @@ -392,7 +216,6 @@ OpticalProps1sclK get_subsampled_clouds( * can be used as-is, but are intended to be wrapped by the SCREAM AD * interface to radiation. */ -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_initialize(GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, @@ -419,65 +242,15 @@ void rrtmgp_initialize(GasConcs &gas_concs, // We are now initialized! initialized = true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger) -{ - // If we've already initialized, just exit - if (initialized_k) { - if (logger) - logger->info("RRTMGP is already initialized; skipping\n"); - return; - } - - // Initialize Kokkos - if (!Kokkos::is_initialized()) { Kokkos::initialize(); } - - // Load and initialize absorption coefficient data - load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); - load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); - - // Load and initialize cloud optical property look-up table information - load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); - load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); - - // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 18000; - const size_t ncol = gas_concs.ncol; - const size_t nlay = gas_concs.nlay; - const size_t nlev = SCREAM_NUM_VERTICAL_LEV; - const size_t my_size_ref = ncol * nlay * nlev; - conv::MemPoolSingleton::init(2e6 * (float(my_size_ref) / base_ref)); - - // We are now initialized! - initialized_k = true; -} -#endif - void rrtmgp_finalize() { -#ifdef RRTMGP_ENABLE_YAKL initialized = false; k_dist_sw.finalize(); k_dist_lw.finalize(); cloud_optics_sw.finalize(); //~CloudOptics(); cloud_optics_lw.finalize(); //~CloudOptics(); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - initialized_k = false; - k_dist_sw_k.finalize(); - k_dist_lw_k.finalize(); - cloud_optics_sw_k.finalize(); //~CloudOptics(); - cloud_optics_lw_k.finalize(); //~CloudOptics(); - conv::MemPoolSingleton::finalize(); -#endif } -#ifdef RRTMGP_ENABLE_YAKL void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, @@ -527,56 +300,7 @@ void compute_band_by_band_surface_albedos( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif) { - - EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - - EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, - "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); - EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), - "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); - - // Loop over bands, and determine for each band whether it is broadly in the - // visible or infrared part of the spectrum (visible or "not visible") - Kokkos::parallel_for(conv::get_mdrp<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); - } - else { - // Band straddles the visible to near-infrared transition, so we take - // the albedo to be the average of the visible and near-infrared - // broadband albedos - sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); - sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , @@ -631,62 +355,7 @@ void compute_broadband_surface_fluxes( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir) { - // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums - // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when - // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This - // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. - //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - - // Initialize sums over bands - Kokkos::deep_copy(sfc_flux_dir_nir, 0); - Kokkos::deep_copy(sfc_flux_dir_vis, 0); - Kokkos::deep_copy(sfc_flux_dif_nir, 0); - Kokkos::deep_copy(sfc_flux_dif_vis, 0); - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { - for (int ibnd = 0; ibnd < nswbands; ++ibnd) { - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else { - // Band straddles the visible to near-infrared transition, so put half - // the flux in visible and half in near-infrared fluxes - sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - } - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -855,191 +524,19 @@ void rrtmgp_main( ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, - real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - -#ifdef SCREAM_RRTMGP_DEBUG - // Sanity check inputs, and possibly repair - check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); - check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); - check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); - check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); - check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); - check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); - check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); - check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); - check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); - check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); - check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); -#endif - - // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; - fluxes_sw.flux_up = sw_flux_up; - fluxes_sw.flux_dn = sw_flux_dn; - fluxes_sw.flux_dn_dir = sw_flux_dn_dir; - fluxes_sw.bnd_flux_up = sw_bnd_flux_up; - fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; - fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; - clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; - clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; - clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; - clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; - clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; - clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; - clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; - clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; - clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; - // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; - fluxes_lw.flux_up = lw_flux_up; - fluxes_lw.flux_dn = lw_flux_dn; - fluxes_lw.bnd_flux_up = lw_bnd_flux_up; - fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; - clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; - clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; - clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; - clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; - clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; - clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; +int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - auto nswbands = k_dist_sw_k.get_nband(); - auto nlwbands = k_dist_lw_k.get_nband(); + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; - aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); - aerosol_sw.alloc_2str(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); - aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); - aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); - }); - aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); - aerosol_lw.alloc_1scl(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Check aerosol optical properties - // NOTE: these should already have been checked by precondition checks, but someday we might have - // non-trivial aerosol optics, so this is still good to do here. - check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); - check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); - check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); - check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); -#endif - - // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); - Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); - Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); - - // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; - // This implements the Monte Carlo Independing Column Approximation by mapping only a single - // subcolumn (cloud state) to each gpoint. - auto nswgpts = k_dist_sw_k.get_ngpt(); - auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); - // Longwave - auto nlwgpts = k_dist_lw_k.get_ngpt(); - auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); - - // Copy cloud properties to outputs (is this needed, or can we just use pointers?) - // Alternatively, just compute and output a subcolumn cloud mask - Kokkos::parallel_for(conv::get_mdrp<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, - // but we might want to provide additional debug info here. NOTE: we may actually want to move this - // up higher in the code, I think optical props should go up higher since optical props are kind of - // a parameterization of their own, and we might want to swap different choices. These checks go here - // only because we need to run them on computed optical props, so if the optical props themselves get - // computed up higher, then perform these checks higher as well - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); - check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); - check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); -#endif - - // Do shortwave - rrtmgp_sw( - ncol, nlay, - k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, - fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, - tsi_scaling, logger, - extra_clnclrsky_diag, extra_clnsky_diag - ); - - // Do longwave - rrtmgp_lw( - ncol, nlay, - k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - aerosol_lw, clouds_lw_gpt, - fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, - extra_clnclrsky_diag, extra_clnsky_diag - ); - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL -int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3d("cldx", ncol, nlay, ngpt); + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = real3d("cldx", ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1090,84 +587,7 @@ int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d }); return subcolumn_mask; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3dk("cldx", ncol, nlay, ngpt); - - // Apply overlap assumption to set cldx - if (overlap_option == 0) { // Dummy mask, always cloudy - Kokkos::deep_copy(cldx, 1); - } else { // Default case, maximum-random overlap - // Maximum-random overlap: - // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, - // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same - // algorithm used in RRTMG implementation of maximum-random overlap (see - // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) - // - // First, fill cldx with random numbers. Need to use a unique seed for each column! - // auto seeds_host = Kokkos::create_mirror_view(seeds); - // Kokkos::deep_copy(seeds_host, seeds); - // for (int icol = 0; icol < ncol; ++icol) { - // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { - // auto generator = random_pool.get_state(); - // cldx(icol,ilay,igpt) = generator.drand(0., 1.); - // random_pool.free_state(generator); - // }); - // } - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - conv::Random rand(seeds(icol)); - for (int igpt = 0; igpt < ngpt; igpt++) { - for (int ilay = 0; ilay < nlay; ilay++) { - cldx(icol,ilay,igpt) = rand.genFP(); - } - } - }); - - // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { - for (int ilay = 1; ilay < nlay; ilay++) { - // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn - if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { - // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent - // layers are maximimally overlapped - cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); - } else { - // Cloud-less above, use new random number so that clouds are distributed - // randomly in this layer. Need to scale new random number to range - // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution - // of random numbers in this layer with the above branch of the conditional, - // which would otherwise inflate cloud fraction in this layer. - cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); - } - } - }); - } - - // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { - subcolumn_mask(icol,ilay,igpt) = 1; - } else { - subcolumn_mask(icol,ilay,igpt) = 0; - } - }); - return subcolumn_mask; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1423,267 +843,7 @@ void rrtmgp_sw( }); } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &flux_dn_dir = fluxes.flux_dn_dir; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; - - // Reset fluxes to zero - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up (icol,ilev) = 0; - flux_dn (icol,ilev) = 0; - flux_dn_dir(icol,ilev) = 0; - clnclrsky_flux_up (icol,ilev) = 0; - clnclrsky_flux_dn (icol,ilev) = 0; - clnclrsky_flux_dn_dir(icol,ilev) = 0; - clrsky_flux_up (icol,ilev) = 0; - clrsky_flux_dn (icol,ilev) = 0; - clrsky_flux_dn_dir(icol,ilev) = 0; - clnsky_flux_up (icol,ilev) = 0; - clnsky_flux_dn (icol,ilev) = 0; - clnsky_flux_dn_dir(icol,ilev) = 0; - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up (icol,ilev,ibnd) = 0; - bnd_flux_dn (icol,ilev,ibnd) = 0; - bnd_flux_dn_dir(icol,ilev,ibnd) = 0; - }); - - // Get daytime indices - auto dayIndices = int1dk("dayIndices", ncol); - Kokkos::deep_copy(dayIndices, -1); - - int nday = 0; - // Serialized for now. - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { - for (int icol = 0; icol < ncol; ++icol) { - if (mu0(icol) > 0) { - dayIndices(nday_inner++) = icol; - } - } - }, Kokkos::Sum(nday)); - - if (nday == 0) { - // No daytime columns in this chunk, skip the rest of this routine - return; - } - - // Subset mu0 - auto mu0_day = real1dk("mu0_day", nday); - Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { - mu0_day(iday) = mu0(dayIndices(iday)); - }); - - // subset state variables - auto p_lay_day = real2dk("p_lay_day", nday, nlay); - auto t_lay_day = real2dk("t_lay_day", nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); - t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); - }); - auto p_lev_day = real2dk("p_lev_day", nday, nlay+1); - auto t_lev_day = real2dk("t_lev_day", nday, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); - t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); - }); - - // Subset gases - auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; - gas_concs_day.init(gas_names, nday, nlay); - for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = real2dk("vmr_day", nday, nlay); - auto vmr = real2dk("vmr" , ncol, nlay); - gas_concs.get_vmr(gas_names[igas], vmr); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); - }); - gas_concs_day.set_vmr(gas_names[igas], vmr_day); - } - - // Subset aerosol optics - OpticalProps2strK aerosol_day; - aerosol_day.init(k_dist.get_band_lims_wavenumber()); - aerosol_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { - aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); - aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); - aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); - }); - - // Subset cloud optics - // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; - clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); - clouds_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { - clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); - clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); - clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); - }); - - // RRTMGP assumes surface albedos have a screwy dimension ordering - // for some strange reason, so we need to transpose these; also do - // daytime subsetting in the same kernel - real2dk sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - real2dk sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); - Kokkos::parallel_for(conv::get_mdrp<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { - sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); - sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); - }); - - // Temporaries we need for daytime-only fluxes - auto flux_up_day = real2dk("flux_up_day", nday, nlay+1); - auto flux_dn_day = real2dk("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = real2dk("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = real3dk("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = real3dk("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = real3dk("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; - fluxes_day.flux_up = flux_up_day; - fluxes_day.flux_dn = flux_dn_day; - fluxes_day.flux_dn_dir = flux_dn_dir_day; - fluxes_day.bnd_flux_up = bnd_flux_up_day; - fluxes_day.bnd_flux_dn = bnd_flux_dn_day; - fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; - - // Allocate space for optical properties - OpticalProps2strK optics; - optics.alloc_2str(nday, nlay, k_dist); - - OpticalProps2strK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_2str(nday, nlay, k_dist); - } - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", nday, nlay); - limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); - - // Do gas optics - real2dk toa_flux("toa_flux", nday, ngpt); - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); - if (extra_clnsky_diag) { - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); - check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); - check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); -#endif - - // Apply tsi_scaling - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { - toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); - }); - - if (extra_clnclrsky_diag) { - // Compute clear-clean-sky (just gas) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } - - // Combine gas and aerosol optics - aerosol_day.delta_scale(); - aerosol_day.increment(optics); - - // Compute clearsky (gas + aerosol) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - - // Now merge in cloud optics and do allsky calculations - - // Combine gas and cloud optics - clouds_day.delta_scale(); - clouds_day.increment(optics); - // Compute fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - flux_up (icol,ilev) = flux_up_day (iday,ilev); - flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { - const int icol = dayIndices(iday); - bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); - bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); - bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); - }); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds_day.increment(optics_no_aerosols); - // Compute cleansky (gas + clouds) fluxes on daytime columns - rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1816,147 +976,7 @@ void rrtmgp_lw( } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, - OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Problem size - int nbnd = k_dist.get_nband(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - - // Reset fluxes to zero - Kokkos::parallel_for( - conv::get_mdrp<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up(icol, ilev) = 0; - flux_dn(icol, ilev) = 0; - clnclrsky_flux_up(icol, ilev) = 0; - clnclrsky_flux_dn(icol, ilev) = 0; - clrsky_flux_up(icol, ilev) = 0; - clrsky_flux_dn(icol, ilev) = 0; - clnsky_flux_up(icol, ilev) = 0; - clnsky_flux_dn(icol, ilev) = 0; - }); - Kokkos::parallel_for( - conv::get_mdrp<3>({nbnd, nlay + 1, ncol}), - KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up(icol, ilev, ibnd) = 0; - bnd_flux_dn(icol, ilev, ibnd) = 0; - }); - // Allocate space for optical properties - OpticalProps1sclK optics; - optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); - } - - // Boundary conditions - SourceFuncLWK lw_sources; - lw_sources.alloc(ncol, nlay, k_dist); - real1dk t_sfc ("t_sfc" ,ncol); - real2dk emis_sfc("emis_sfc",nbnd,ncol); - - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - // Surface temperature - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); - }); - Kokkos::deep_copy(emis_sfc , 0.98); - - // Get Gaussian quadrature weights - // TODO: move this crap out of userland! - // Weights and angle secants for first order (k=1) Gaussian quadrature. - // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 - // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - realHost2dk gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - realHost2dk gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - real2dk gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - real2dk gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); - Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); - Kokkos::deep_copy(gauss_wts, gauss_wts_host); - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", ncol, nlay); - auto t_lev_limited = real2dk("t_lev_limited", ncol, nlay+1); - limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); - limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); - - // Do gas optics - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, real2dk(), t_lev_limited); - if (extra_clnsky_diag) { - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, real2dk(), t_lev_limited); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); -#endif - - if (extra_clnclrsky_diag) { - // Compute clean-clear-sky fluxes before we add in aerosols and clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); - } - - // Combine gas and aerosol optics - aerosol.increment(optics); - - // Compute clear-sky fluxes before we add in clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); - - // Combine gas and cloud optics - clouds.increment(optics); - - // Compute allsky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds.increment(optics_no_aerosols); - // Compute clean-sky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); - } - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area) { @@ -1981,46 +1001,11 @@ void compute_cloud_area( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area) { - // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy - // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = real2dk("subcol_mask", ncol, ngpt); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but - // using play/pmid does not - if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { - subcol_mask(icol,igpt) = 1; - } - }); - // Compute average over subcols to get cloud area - auto ngpt_inv = 1.0 / ngpt; - Kokkos::deep_copy(cld_area, 0); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // This loop needs to be serial because of the atomic reduction - for (int igpt = 0; igpt < ngpt; ++igpt) { - cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index_sw_k(double wavelength) { return get_wavelength_index(k_dist_sw_k, wavelength); } - -int get_wavelength_index_lw_k(double wavelength) { return get_wavelength_index(k_dist_lw_k, wavelength); } -#endif - -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index(OpticalProps &kdist, double wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -2043,33 +1028,7 @@ int get_wavelength_index(OpticalProps &kdist, double wavelength) { }); return band_index.hostRead(); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength) { - // Get wavelength bounds for all wavelength bands - auto wavelength_bounds = kdist.get_band_lims_wavelength(); - - // Find the band index for the specified wavelength - // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength - // in units of meters, we need a conversion factor of 10^2 - const int nbnds = kdist.get_nband(); - int band_index = -1; - Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { - if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { - if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } else { - if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } - }, Kokkos::Max(band_index)); - return band_index; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -2166,109 +1125,7 @@ void compute_aerocom_cloudtop( cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop) { - /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCom recommendation. See reference for routine - * get_subcolumn_mask above, where equation 14 is used for the - * maximum-random overlap assumption for subcolumn generation. We use - * equation 13, the column counterpart. - */ - // Set outputs to zero - Kokkos::deep_copy(T_mid_at_cldtop, 0.0); - Kokkos::deep_copy(p_mid_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); - Kokkos::deep_copy(cdnc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); - - // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = real1dk("aerocom_clr", ncol); - Kokkos::deep_copy(aerocom_clr, 1.0); - - // Get gravity acceleration constant from constants - using physconst = scream::physics::Constants; - - // TODO: move tunable constant to namelist - constexpr real q_threshold = 0.0; // BAD_CONSTANT! - - // TODO: move tunable constant to namelist - constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! - - // Loop over all columns in parallel - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // Loop over all layers in serial (due to accumulative - // product), starting at 2 (second highest) layer because the - // highest is assumed to hav no clouds - for(int ilay = 1; ilay < nlay; ++ilay) { - // Only do the calculation if certain conditions are met - if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && - (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { - /* PART I: Probabilistically determining cloud top */ - // Populate aerocom_tmp as the clear-sky fraction - // probability of this level, where aerocom_clr is that of - // the previous level - auto aerocom_tmp = - aerocom_clr(icol) * - (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), - cldfrac_tot(icol, ilay))) / - (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), - 1.0 - cldfrac_tot_threshold)); - // Temporary variable for probability "weights" - auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; - // Temporary variable for liquid "phase" - auto aerocom_phi = - qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); - /* PART II: The inferred properties */ - /* In general, converting a 3D property X to a 2D cloud-top - * counterpart x follows: x(i) += X(i,k) * weights * Phase - * but X and Phase are not always needed */ - // T_mid_at_cldtop - T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; - // p_mid_at_cldtop - p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; - // cldfrac_ice_at_cldtop - cldfrac_ice_at_cldtop(icol) += - (1.0 - aerocom_phi) * aerocom_wts; - // cldfrac_liq_at_cldtop - cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; - // cdnc_at_cldtop - /* We need to convert nc from 1/mass to 1/volume first, and - * from grid-mean to in-cloud, but after that, the - * calculation follows the general logic */ - auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / - z_del(icol, ilay) / physconst::gravit / - cldfrac_tot(icol, ilay); - cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; - // eff_radius_qc_at_cldtop - eff_radius_qc_at_cldtop(icol) += - rel(icol, ilay) * aerocom_phi * aerocom_wts; - // eff_radius_qi_at_cldtop - eff_radius_qi_at_cldtop(icol) += - rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset aerocom_clr to aerocom_tmp to accumulate - aerocom_clr(icol) = aerocom_tmp; - } - } - // After the serial loop over levels, the cloudy fraction is - // defined as (1 - aerocom_clr). This is true because - // aerocom_clr is the result of accumulative probabilities - // (their products) - cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); - }); -} -#endif } // namespace rrtmgp +#endif } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 49fbfeb4486b..c40be4904302 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -6,10 +6,17 @@ #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" #include "cpp/rrtmgp_const.h" +#include "rrtmgp_utils.hpp" + #include "physics/share/physics_constants.hpp" #include "ekat/mpi/ekat_comm.hpp" #include "ekat/logging/ekat_logger.hpp" +#include "ekat/util/ekat_math_utils.hpp" + +#ifdef RRTMGP_ENABLE_KOKKOS +#include "Kokkos_Random.hpp" +#endif namespace scream { @@ -18,105 +25,34 @@ void finalize_kls(); namespace rrtmgp { -/* - * Objects containing k-distribution information need to be initialized - * once and then persist throughout the life of the program, so we - * declare them here within the rrtmgp namespace. - */ #ifdef RRTMGP_ENABLE_YAKL extern GasOpticsRRTMGP k_dist_sw; extern GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern GasOpticsRRTMGPK k_dist_sw_k; -extern GasOpticsRRTMGPK k_dist_lw_k; -#endif -/* - * Objects containing cloud optical property look-up table information. - * We want to initialize these once and use throughout the life of the - * program, so declare here and read data in during rrtmgp_initialize(). - */ -#ifdef RRTMGP_ENABLE_YAKL extern CloudOptics cloud_optics_sw; extern CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern CloudOpticsK cloud_optics_sw_k; -extern CloudOpticsK cloud_optics_lw_k; -#endif -/* - * Flag to indicate whether or not we have initialized RRTMGP - */ -#ifdef RRTMGP_ENABLE_YAKL extern bool initialized; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern bool initialized_k; -#endif -/* - * Initialize data for RRTMGP driver - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_initialize( +void rrtmgp_initialize( GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger); -#endif -/* - * Compute band-by-band surface albedos from broadband albedos. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_band_by_band_surface_albedos( +void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, real2d &sfc_alb_dir, real2d &sfc_alb_dif); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif); -#endif -/* - * Compute broadband visible/UV and near-infrared surface fluxes. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_broadband_surface_fluxes( +void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir); -#endif -/* - * Main driver code to run RRTMGP. - * The input logger is in charge of outputing info to - * screen and/or to file (or neither), depending on how it was set up. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_main( +void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, GasConcs &gas_concs, @@ -138,42 +74,10 @@ extern void rrtmgp_main( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -/* - * Perform any clean-up tasks - */ -extern void rrtmgp_finalize(); +void rrtmgp_finalize(); -/* - * Shortwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_sw( +void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -184,26 +88,8 @@ extern void rrtmgp_sw( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Longwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_lw( +void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -211,46 +97,13 @@ extern void rrtmgp_lw( OpticalProps1scl &aerosol, OpticalProps1scl &clouds, FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Return a subcolumn mask consistent with a specified overlap assumption - */ -#ifdef RRTMGP_ENABLE_YAKL int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds); -#endif -/* - * Compute cloud area from 3d subcol cloud property - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, Real pmin, Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, Real pmin, Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area); -#endif -/* - * Return select cloud-top diagnostics following AeroCom recommendation - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -260,26 +113,7 @@ void compute_aerocom_cloudtop( real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop); -#endif -/* - * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area - * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion - * with the standard practice of using "water path" to refer to the total column-integrated - * quantities). - */ -#ifdef RRTMGP_ENABLE_YAKL template void mixing_ratio_to_cloud_mass( yakl::Array const &mixing_ratio, @@ -303,10 +137,1035 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif + +template +void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { + yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { + arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + }); +} + +int get_wavelength_index(OpticalProps &kdist, double wavelength); +int get_wavelength_index_sw(double wavelength); +int get_wavelength_index_lw(double wavelength); +#endif // RRTMGP_ENABLE_YAKL + +// New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS +template +struct rrtmgp_interface { + +using MDRP = typename conv::MDRP; + +template +using view_t = Kokkos::View; + +template +using oview_t = Kokkos::Experimental::OffsetView; + +template +using hview_t = Kokkos::View; + +using pool_t = conv::MemPoolSingleton; + +/* + * Objects containing k-distribution information need to be initialized + * once and then persist throughout the life of the program, so we + * declare them here within the rrtmgp namespace. + */ +static inline GasOpticsRRTMGPK k_dist_sw_k; +static inline GasOpticsRRTMGPK k_dist_lw_k; + +/* + * Objects containing cloud optical property look-up table information. + * We want to initialize these once and use throughout the life of the + * program, so declare here and read data in during rrtmgp_initialize(). + */ +static inline CloudOpticsK cloud_optics_sw_k; +static inline CloudOpticsK cloud_optics_lw_k; + +/* + * Flag to indicate whether or not we have initialized RRTMGP + */ +static inline bool initialized_k = false; + +/* + * Initialize data for RRTMGP driver + */ +static void rrtmgp_initialize( + GasConcsK &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger) +{ + // If we've already initialized, just exit + if (initialized_k) { + if (logger) + logger->info("RRTMGP is already initialized; skipping\n"); + return; + } + + // Initialize Kokkos + if (!Kokkos::is_initialized()) { Kokkos::initialize(); } + + // Load and initialize absorption coefficient data + load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); + load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); + + // Load and initialize cloud optical property look-up table information + load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); + load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); + + // initialize kokkos rrtmgp pool allocator + const size_t base_ref = 18000; + const size_t ncol = gas_concs.ncol; + const size_t nlay = gas_concs.nlay; + const size_t nlev = SCREAM_NUM_VERTICAL_LEV; + const size_t my_size_ref = ncol * nlay * nlev; + pool_t::init(2e6 * (float(my_size_ref) / base_ref)); + + // We are now initialized! + initialized_k = true; +} + +/* + * Compute band-by-band surface albedos from broadband albedos. + */ +template +static void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + SfcAlbDirVisT &sfc_alb_dir_vis, SfcAlbDirNirT &sfc_alb_dir_nir, + SfcAlbDifVisT &sfc_alb_dif_vis, SfcAlbDifNirT &sfc_alb_dif_nir, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif) +{ + EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + + EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, + "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); + EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), + "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); + + // Loop over bands, and determine for each band whether it is broadly in the + // visible or infrared part of the spectrum (visible or "not visible") + Kokkos::parallel_for(MDRP::template get<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); + } + else { + // Band straddles the visible to near-infrared transition, so we take + // the albedo to be the average of the visible and near-infrared + // broadband albedos + sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); + sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); + } + }); +} + +/* + * Compute broadband visible/UV and near-infrared surface fluxes. + */ +template +static void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + SwBndFluxDirT &sw_bnd_flux_dir , SwBndFluxDifT &sw_bnd_flux_dif , + SfcFluxDirVisT &sfc_flux_dir_vis, SfcFluxDirNirT &sfc_flux_dir_nir, + SfcFluxDifVisT &sfc_flux_dif_vis, SfcFluxDifNirT &sfc_flux_dif_nir) +{ + // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums + // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when + // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This + // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. + //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + + // Initialize sums over bands + Kokkos::deep_copy(sfc_flux_dir_nir, 0); + Kokkos::deep_copy(sfc_flux_dir_vis, 0); + Kokkos::deep_copy(sfc_flux_dif_nir, 0); + Kokkos::deep_copy(sfc_flux_dif_vis, 0); + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { + for (int ibnd = 0; ibnd < nswbands; ++ibnd) { + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else { + // Band straddles the visible to near-infrared transition, so put half + // the flux in visible and half in near-infrared fluxes + sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + } + } + }); +} + +/* + * Main driver code to run RRTMGP. + * The input logger is in charge of outputing info to + * screen and/or to file (or neither), depending on how it was set up. + */ +template +static void rrtmgp_main( + const int ncol, const int nlay, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei, CldfracT &cldfrac, + AerTauSwT &aer_tau_sw, AerSsaSwT &aer_ssa_sw, AerAsmSwT &aer_asm_sw, AerTauLwT &aer_tau_lw, + CldTauSwBndT &cld_tau_sw_bnd, CldTauLwBndT &cld_tau_lw_bnd, + CldTauSwGptT &cld_tau_sw_gpt, CldTauLwGptT &cld_tau_lw_gpt, + SwFluxUpT &sw_flux_up, SwFluxDnT &sw_flux_dn, SwFluxDnDirT &sw_flux_dn_dir, + LwFluxUpT &lw_flux_up, LwFluxDnT &lw_flux_dn, + SwClnclrskyFluxUpT &sw_clnclrsky_flux_up, SwClnclrskyFluxDnT &sw_clnclrsky_flux_dn, SwClnclrskyFluxDnDirT &sw_clnclrsky_flux_dn_dir, + SwClrskyFluxUpT &sw_clrsky_flux_up, SwClrskyFluxDnT &sw_clrsky_flux_dn, SwClrskyFluxDnDirT &sw_clrsky_flux_dn_dir, + SwClnskyFluxUpT &sw_clnsky_flux_up, SwClnskyFluxDnT &sw_clnsky_flux_dn, SwClnskyFluxDnDirT &sw_clnsky_flux_dn_dir, + LwClnclrskyFluxUpT &lw_clnclrsky_flux_up, LwClnclrskyFluxDnT &lw_clnclrsky_flux_dn, + LwClrskyFluxUpT &lw_clrsky_flux_up, LwClrskyFluxDnT &lw_clrsky_flux_dn, + LwClnskyFluxUpT &lw_clnsky_flux_up, LwClnskyFluxDnT &lw_clnsky_flux_dn, + SwBndFluxUpT &sw_bnd_flux_up, SwBndFluxDnT &sw_bnd_flux_dn, SwBndFluxDnDirT &sw_bnd_flux_dn_dir, + LwBndFluxUpT &lw_bnd_flux_up, LwBndFluxDnT &lw_bnd_flux_dn, + const RealT tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) +{ +#ifdef SCREAM_RRTMGP_DEBUG + // Sanity check inputs, and possibly repair + check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); + check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); + check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); + check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); + check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); + check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); + check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); + check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); + check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); + check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); + check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); +#endif + + // Setup pointers to RRTMGP SW fluxes + FluxesBybandK fluxes_sw; + fluxes_sw.flux_up = sw_flux_up; + fluxes_sw.flux_dn = sw_flux_dn; + fluxes_sw.flux_dn_dir = sw_flux_dn_dir; + fluxes_sw.bnd_flux_up = sw_bnd_flux_up; + fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; + fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_sw; + clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; + clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; + clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_sw; + clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; + clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; + clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_sw; + clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; + clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; + clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; + + // Setup pointers to RRTMGP LW fluxes + FluxesBybandK fluxes_lw; + fluxes_lw.flux_up = lw_flux_up; + fluxes_lw.flux_dn = lw_flux_dn; + fluxes_lw.bnd_flux_up = lw_bnd_flux_up; + fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_lw; + clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; + clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_lw; + clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; + clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_lw; + clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; + clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; + + auto nswbands = k_dist_sw_k.get_nband(); + auto nlwbands = k_dist_lw_k.get_nband(); + + // Setup aerosol optical properties + OpticalProps2strK aerosol_sw; + OpticalProps1sclK aerosol_lw; + aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); + aerosol_sw.alloc_2str(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); + aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); + aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); + }); + aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); + aerosol_lw.alloc_1scl(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Check aerosol optical properties + // NOTE: these should already have been checked by precondition checks, but someday we might have + // non-trivial aerosol optics, so this is still good to do here. + check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); + check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); + check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); + check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); +#endif + + // Convert cloud physical properties to optical properties for input to RRTMGP + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); + Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); + + // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; + // This implements the Monte Carlo Independing Column Approximation by mapping only a single + // subcolumn (cloud state) to each gpoint. + auto nswgpts = k_dist_sw_k.get_ngpt(); + auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); + // Longwave + auto nlwgpts = k_dist_lw_k.get_ngpt(); + auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); + + // Copy cloud properties to outputs (is this needed, or can we just use pointers?) + // Alternatively, just compute and output a subcolumn cloud mask + Kokkos::parallel_for(MDRP::template get<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); + }); + Kokkos::parallel_for(MDRP::template get<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, + // but we might want to provide additional debug info here. NOTE: we may actually want to move this + // up higher in the code, I think optical props should go up higher since optical props are kind of + // a parameterization of their own, and we might want to swap different choices. These checks go here + // only because we need to run them on computed optical props, so if the optical props themselves get + // computed up higher, then perform these checks higher as well + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); + check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); +#endif + + // Do shortwave + rrtmgp_sw( + ncol, nlay, + k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, + fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, + tsi_scaling, logger, + extra_clnclrsky_diag, extra_clnsky_diag + ); + + // Do longwave + rrtmgp_lw( + ncol, nlay, + k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + aerosol_lw, clouds_lw_gpt, + fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, + extra_clnclrsky_diag, extra_clnsky_diag + ); + +} + +/* + * Perform any clean-up tasks + */ +static void rrtmgp_finalize() +{ + initialized_k = false; + k_dist_sw_k.finalize(); + k_dist_lw_k.finalize(); + cloud_optics_sw_k.finalize(); //~CloudOptics(); + cloud_optics_lw_k.finalize(); //~CloudOptics(); + pool_t::finalize(); +} + +/* + * Shortwave driver (called by rrtmgp_main) + */ +template +static void rrtmgp_sw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, + OpticalProps2strK &aerosol, OpticalProps2strK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const RealT tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Get problem sizes + int nbnd = k_dist.get_nband(); + int ngpt = k_dist.get_ngpt(); + int ngas = gas_concs.get_num_gases(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &flux_dn_dir = fluxes.flux_dn_dir; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; + + // Reset fluxes to zero + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up (icol,ilev) = 0; + flux_dn (icol,ilev) = 0; + flux_dn_dir(icol,ilev) = 0; + clnclrsky_flux_up (icol,ilev) = 0; + clnclrsky_flux_dn (icol,ilev) = 0; + clnclrsky_flux_dn_dir(icol,ilev) = 0; + clrsky_flux_up (icol,ilev) = 0; + clrsky_flux_dn (icol,ilev) = 0; + clrsky_flux_dn_dir(icol,ilev) = 0; + clnsky_flux_up (icol,ilev) = 0; + clnsky_flux_dn (icol,ilev) = 0; + clnsky_flux_dn_dir(icol,ilev) = 0; + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up (icol,ilev,ibnd) = 0; + bnd_flux_dn (icol,ilev,ibnd) = 0; + bnd_flux_dn_dir(icol,ilev,ibnd) = 0; + }); + + // Get daytime indices + auto dayIndices = view_t("dayIndices", ncol); + Kokkos::deep_copy(dayIndices, -1); + + int nday = 0; + // Serialized for now. + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { + for (int icol = 0; icol < ncol; ++icol) { + if (mu0(icol) > 0) { + dayIndices(nday_inner++) = icol; + } + } + }, Kokkos::Sum(nday)); + + if (nday == 0) { + // No daytime columns in this chunk, skip the rest of this routine + return; + } + + // Subset mu0 + auto mu0_day = view_t("mu0_day", nday); + Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { + mu0_day(iday) = mu0(dayIndices(iday)); + }); + + // subset state variables + auto p_lay_day = view_t("p_lay_day", nday, nlay); + auto t_lay_day = view_t("t_lay_day", nday, nlay); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); + t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); + }); + auto p_lev_day = view_t("p_lev_day", nday, nlay+1); + auto t_lev_day = view_t("t_lev_day", nday, nlay+1); + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); + t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); + }); + + // Subset gases + auto gas_names = gas_concs.get_gas_names(); + GasConcsK gas_concs_day; + gas_concs_day.init(gas_names, nday, nlay); + for (int igas = 0; igas < ngas; igas++) { + auto vmr_day = view_t("vmr_day", nday, nlay); + auto vmr = view_t("vmr" , ncol, nlay); + gas_concs.get_vmr(gas_names[igas], vmr); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); + }); + gas_concs_day.set_vmr(gas_names[igas], vmr_day); + } + + // Subset aerosol optics + OpticalProps2strK aerosol_day; + aerosol_day.init(k_dist.get_band_lims_wavenumber()); + aerosol_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { + aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); + aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); + aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); + }); + + // Subset cloud optics + // TODO: nbnd -> ngpt once we pass sub-sampled cloud state + OpticalProps2strK clouds_day; + clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); + clouds_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { + clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); + clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); + clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); + }); + + // RRTMGP assumes surface albedos have a screwy dimension ordering + // for some strange reason, so we need to transpose these; also do + // daytime subsetting in the same kernel + view_t sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); + view_t sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); + Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { + sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); + sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); + }); + + // Temporaries we need for daytime-only fluxes + auto flux_up_day = view_t("flux_up_day", nday, nlay+1); + auto flux_dn_day = view_t("flux_dn_day", nday, nlay+1); + auto flux_dn_dir_day = view_t("flux_dn_dir_day", nday, nlay+1); + auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); + FluxesBybandK fluxes_day; + fluxes_day.flux_up = flux_up_day; + fluxes_day.flux_dn = flux_dn_day; + fluxes_day.flux_dn_dir = flux_dn_dir_day; + fluxes_day.bnd_flux_up = bnd_flux_up_day; + fluxes_day.bnd_flux_dn = bnd_flux_dn_day; + fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; + + // Allocate space for optical properties + OpticalProps2strK optics; + optics.alloc_2str(nday, nlay, k_dist); + + OpticalProps2strK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_2str(nday, nlay, k_dist); + } + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = view_t("t_lay_limited", nday, nlay); + limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); + + // Do gas optics + view_t toa_flux("toa_flux", nday, ngpt); + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); + if (extra_clnsky_diag) { + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); + check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); + check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); +#endif + + // Apply tsi_scaling + Kokkos::parallel_for(MDRP::template get<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { + toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); + }); + + if (extra_clnclrsky_diag) { + // Compute clear-clean-sky (just gas) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + + // Combine gas and aerosol optics + aerosol_day.delta_scale(); + aerosol_day.increment(optics); + + // Compute clearsky (gas + aerosol) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + + // Now merge in cloud optics and do allsky calculations + + // Combine gas and cloud optics + clouds_day.delta_scale(); + clouds_day.increment(optics); + // Compute fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + flux_up (icol,ilev) = flux_up_day (iday,ilev); + flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { + const int icol = dayIndices(iday); + bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); + bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); + bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); + }); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds_day.increment(optics_no_aerosols); + // Compute cleansky (gas + clouds) fluxes on daytime columns + rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } +} + +/* + * Longwave driver (called by rrtmgp_main) + */ +template +static void rrtmgp_lw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, + GasConcsK &gas_concs, + OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Problem size + int nbnd = k_dist.get_nband(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + + // Reset fluxes to zero + Kokkos::parallel_for( + MDRP::template get<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up(icol, ilev) = 0; + flux_dn(icol, ilev) = 0; + clnclrsky_flux_up(icol, ilev) = 0; + clnclrsky_flux_dn(icol, ilev) = 0; + clrsky_flux_up(icol, ilev) = 0; + clrsky_flux_dn(icol, ilev) = 0; + clnsky_flux_up(icol, ilev) = 0; + clnsky_flux_dn(icol, ilev) = 0; + }); + Kokkos::parallel_for( + MDRP::template get<3>({nbnd, nlay + 1, ncol}), + KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up(icol, ilev, ibnd) = 0; + bnd_flux_dn(icol, ilev, ibnd) = 0; + }); + + // Allocate space for optical properties + OpticalProps1sclK optics; + optics.alloc_1scl(ncol, nlay, k_dist); + OpticalProps1sclK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); + } + + // Boundary conditions + SourceFuncLWK lw_sources; + lw_sources.alloc(ncol, nlay, k_dist); + view_t t_sfc ("t_sfc" ,ncol); + view_t emis_sfc("emis_sfc",nbnd,ncol); + + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + // Surface temperature + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); + }); + Kokkos::deep_copy(emis_sfc , 0.98); + + // Get Gaussian quadrature weights + // TODO: move this crap out of userland! + // Weights and angle secants for first order (k=1) Gaussian quadrature. + // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 + // after Abramowitz & Stegun 1972, page 921 + int constexpr max_gauss_pts = 4; + hview_t gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; + gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; + gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; + gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; + + hview_t gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); + gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; + gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; + gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; + gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; + + view_t gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + view_t gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); + Kokkos::deep_copy(gauss_wts, gauss_wts_host); + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = view_t("t_lay_limited", ncol, nlay); + auto t_lev_limited = view_t("t_lev_limited", ncol, nlay+1); + limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); + limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); + + // Do gas optics + oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); + if (extra_clnsky_diag) { + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); +#endif + + if (extra_clnclrsky_diag) { + // Compute clean-clear-sky fluxes before we add in aerosols and clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); + } + + // Combine gas and aerosol optics + aerosol.increment(optics); + + // Compute clear-sky fluxes before we add in clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); + + // Combine gas and cloud optics + clouds.increment(optics); + + // Compute allsky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds.increment(optics_no_aerosols); + // Compute clean-sky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); + } +} + +/* + * Return a subcolumn mask consistent with a specified overlap assumption + */ +template +static view_t get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, + CldfT &cldf, const int overlap_option, SeedsT &seeds) +{ + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = view_t("subcolumn_mask", ncol, nlay, ngpt); + + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = view_t("cldx", ncol, nlay, ngpt); + + // Apply overlap assumption to set cldx + if (overlap_option == 0) { // Dummy mask, always cloudy + Kokkos::deep_copy(cldx, 1); + } else { // Default case, maximum-random overlap + // Maximum-random overlap: + // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, + // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same + // algorithm used in RRTMG implementation of maximum-random overlap (see + // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) + // + // First, fill cldx with random numbers. Need to use a unique seed for each column! + // auto seeds_host = Kokkos::create_mirror_view(seeds); + // Kokkos::deep_copy(seeds_host, seeds); + // for (int icol = 0; icol < ncol; ++icol) { + // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); + // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // auto generator = random_pool.get_state(); + // cldx(icol,ilay,igpt) = generator.drand(0., 1.); + // random_pool.free_state(generator); + // }); + // } + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + conv::Random rand(seeds(icol)); + for (int igpt = 0; igpt < ngpt; igpt++) { + for (int ilay = 0; ilay < nlay; ilay++) { + cldx(icol,ilay,igpt) = rand.genFP(); + } + } + }); + + // Step down columns and apply algorithm from eq (14) + Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + for (int ilay = 1; ilay < nlay; ilay++) { + // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn + if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { + // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent + // layers are maximimally overlapped + cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); + } else { + // Cloud-less above, use new random number so that clouds are distributed + // randomly in this layer. Need to scale new random number to range + // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution + // of random numbers in this layer with the above branch of the conditional, + // which would otherwise inflate cloud fraction in this layer. + cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); + } + } + }); + } + + // Use cldx array to create subcolumn mask + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { + subcolumn_mask(icol,ilay,igpt) = 1; + } else { + subcolumn_mask(icol,ilay,igpt) = 0; + } + }); + return subcolumn_mask; +} + +/* + * Compute cloud area from 3d subcol cloud property + */ +template +static void compute_cloud_area( + int ncol, int nlay, int ngpt, RealT pmin, RealT pmax, + const PmidT& pmid, const CldTauGptT& cld_tau_gpt, CldAreaT& cld_area) +{ + // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy + // then 2d subcol mask is 1, otherwise it is 0 + auto subcol_mask = view_t("subcol_mask", ncol, ngpt); + Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but + // using play/pmid does not + if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { + subcol_mask(icol,igpt) = 1; + } + }); + // Compute average over subcols to get cloud area + auto ngpt_inv = 1.0 / ngpt; + Kokkos::deep_copy(cld_area, 0); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // This loop needs to be serial because of the atomic reduction + for (int igpt = 0; igpt < ngpt; ++igpt) { + cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; + } + }); +} + +/* + * Return select cloud-top diagnostics following AeroCom recommendation + */ +template +static void compute_aerocom_cloudtop( + int ncol, int nlay, const TmidT &tmid, const PmidT &pmid, + const PdelT &p_del, const ZdelT &z_del, const QcT &qc, + const QiT &qi, const RelT &rel, const ReiT &rei, + const CldfracTotT &cldfrac_tot, const NcT &nc, + TmidAtCldtopT &T_mid_at_cldtop, PmidAtCldtopT &p_mid_at_cldtop, + CldfracIceAtCldtopT &cldfrac_ice_at_cldtop, CldfracLiqAtCldtopT &cldfrac_liq_at_cldtop, + CldfracTotAtCldtopT &cldfrac_tot_at_cldtop, CdncAtCldtopT &cdnc_at_cldtop, + EffRadiusQcAtCldtopT &eff_radius_qc_at_cldtop, EffRadiusQiAtCldTopT &eff_radius_qi_at_cldtop) +{ + /* The goal of this routine is to calculate properties at cloud top + * based on the AeroCom recommendation. See reference for routine + * get_subcolumn_mask above, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. + */ + // Set outputs to zero + Kokkos::deep_copy(T_mid_at_cldtop, 0.0); + Kokkos::deep_copy(p_mid_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); + Kokkos::deep_copy(cdnc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); + + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto aerocom_clr = view_t("aerocom_clr", ncol); + Kokkos::deep_copy(aerocom_clr, 1.0); + + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + + // TODO: move tunable constant to namelist + constexpr RealT q_threshold = 0.0; // BAD_CONSTANT! + + // TODO: move tunable constant to namelist + constexpr RealT cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! + + // Loop over all columns in parallel + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds + for(int ilay = 1; ilay < nlay; ++ilay) { + // Only do the calculation if certain conditions are met + if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && + (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + aerocom_clr(icol) * + (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), + cldfrac_tot(icol, ilay))) / + (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = + qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; + // p_mid_at_cldtop + p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + cldfrac_ice_at_cldtop(icol) += + (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / + z_del(icol, ilay) / physconst::gravit / + cldfrac_tot(icol, ilay); + cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtop + eff_radius_qc_at_cldtop(icol) += + rel(icol, ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtop + eff_radius_qi_at_cldtop(icol) += + rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + aerocom_clr(icol) = aerocom_tmp; + } + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); + }); +} + +/* + * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area + * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion + * with the standard practice of using "water path" to refer to the total column-integrated + * quantities). + */ template -void mixing_ratio_to_cloud_mass( +static void mixing_ratio_to_cloud_mass( View1 const& mixing_ratio, View2 const& cloud_fraction, View3 const& dp, @@ -315,7 +1174,7 @@ void mixing_ratio_to_cloud_mass( int ncol = mixing_ratio.extent(0); int nlay = mixing_ratio.extent(1); using physconst = scream::physics::Constants; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { // Compute in-cloud mixing ratio (mixing ratio of the cloudy part of the layer) // NOTE: these thresholds (from E3SM) seem arbitrary, but included here for consistency // This limits in-cloud mixing ratio to 0.005 kg/kg. According to note in cloud_diagnostics @@ -329,7 +1188,6 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif /* * Routine to limit a quantity to set bounds. Used to make sure @@ -337,33 +1195,199 @@ void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -#ifdef RRTMGP_ENABLE_YAKL template -void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { - yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { +static void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { + Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -template -void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + +static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) +{ + // Get wavelength bounds for all wavelength bands + auto wavelength_bounds = kdist.get_band_lims_wavelength(); + + // Find the band index for the specified wavelength + // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength + // in units of meters, we need a conversion factor of 10^2 + const int nbnds = kdist.get_nband(); + int band_index = -1; + Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { + if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { + if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } else { + if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } + }, Kokkos::Max(band_index)); + return band_index; +} + +static inline int get_wavelength_index_sw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_sw_k, wavelength); +} + +static inline int get_wavelength_index_lw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_lw_k, wavelength); +} + +template +static OpticalProps2strK get_cloud_optics_sw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + + // Initialize optics + OpticalProps2strK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_2str(ncol, nlay); + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = view_t("rel_limited", ncol, nlay); + auto rei_limited = view_t("rei_limited", ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} + +template +static OpticalProps1sclK get_cloud_optics_lw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + + // Initialize optics + OpticalProps1sclK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = view_t("rel_limited", ncol, nlay); + auto rei_limited = view_t("rei_limited", ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} + +template +OpticalProps2strK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { + // Initialized subsampled optics + OpticalProps2strK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_2str(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, + // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), + // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should + // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very + // high resolution. + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM + auto seeds = view_t("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); + subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + subsampled_optics.ssa(icol,ilay,igpt) = 0; + subsampled_optics.g (icol,ilay,igpt) = 0; + } }); + return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_YAKL -int get_wavelength_index(OpticalProps &kdist, double wavelength); -int get_wavelength_index_sw(double wavelength); -int get_wavelength_index_lw(double wavelength); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength); -int get_wavelength_index_sw_k(double wavelength); -int get_wavelength_index_lw_k(double wavelength); -#endif +template +OpticalProps1sclK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, + CldT &cld, PlayT &p_lay) { + // Initialized subsampled optics + OpticalProps1sclK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_1scl(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM; use different + // seed values for longwave and shortwave + auto seeds = view_t("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + } + }); + return subsampled_optics; +} + +}; // struct rrtmgp_interface +#endif // RRTMGP_ENABLE_KOKKOS } // namespace rrtmgp } // namespace scream From 8677a1f45b071573f6142c46cdbc941048ab0fc9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 6 Jun 2024 16:55:23 -0600 Subject: [PATCH 248/477] Builds and runs --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 109 +++++----- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 130 ++++++------ .../src/physics/rrtmgp/rrtmgp_test_utils.cpp | 200 +----------------- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 128 ++++++++++- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 5 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 7 - .../rrtmgp/scream_rrtmgp_interface.hpp | 33 +-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 68 +++--- 9 files changed, 306 insertions(+), 376 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 59e6f72cc922..3d7fba5ac72d 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 59e6f72cc92268c69d1bd6ede570a959c1ebc28a +Subproject commit 3d7fba5ac72dab7c8448411e7452e2b409f22d93 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1ff3f27b2fce..a24f4d5ddc32 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -624,17 +624,17 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.init(gas_names_yakl_offset,m_col_chunk_size,m_nlay); - rrtmgp::rrtmgp_initialize( + interface_t::rrtmgp_initialize( m_gas_concs_k, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, m_atm_logger ); VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_sw, rrtmgp::k_dist_sw_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_lw, rrtmgp::k_dist_lw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, rrtmgp::cloud_optics_sw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, rrtmgp::cloud_optics_lw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_sw, interface_t::k_dist_sw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_lw, interface_t::k_dist_lw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, interface_t::cloud_optics_sw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, interface_t::cloud_optics_lw_k); #endif // Set property checks for fields in this process @@ -924,30 +924,30 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS // If YAKL is on, we don't want aliased memory in both the yakl and kokos // subviews. - auto subview_1dk = [&](const real1dk v) -> real1dk { - real1dk subv(v, std::make_pair(0, ncol)); + auto subview_1dk = [&](const view_1d_real v) -> view_1d_real { + view_1d_real subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - real1dk rv(v.label(), ncol); + view_1d_real rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_2dk = [&](const real2dk v) -> real2dk { - real2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_2dk = [&](const view_2d_real v) -> view_2d_real { + view_2d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); + view_2d_real rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_3dk = [&](const real3dk v) -> real3dk { - real3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + auto subview_3dk = [&](const view_3d_real v) -> view_3d_real { + view_3d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); + view_3d_real rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); return rv; #else @@ -1194,7 +1194,7 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::fence(); #ifdef RRTMGP_ENABLE_KOKKOS COMPARE_ALL_WRAP(std::vector({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1206,7 +1206,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real2d tmp2d = subview_2d(m_buffer.tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); + view_2d_real tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1308,14 +1308,14 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + interface_t::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + interface_t::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); + interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); + interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), - std::vector({lwp_k, iwp_k})); + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1350,13 +1350,13 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dir, sfc_alb_dif); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis_k, sfc_alb_dir_nir_k, sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), - std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1384,7 +1384,7 @@ void RRTMGPRadiation::run_impl (const double dt) { ); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, m_nlay, p_lay_k, t_lay_k, p_lev_k, t_lev_k, m_gas_concs_k, @@ -1412,7 +1412,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, lw_clnsky_flux_up, lw_clnsky_flux_dn}), - std::vector({ + std::vector({ sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, @@ -1422,7 +1422,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), - std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1472,7 +1472,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), - std::vector({sw_heating_k, lw_heating_k})); + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1509,14 +1509,14 @@ void RRTMGPRadiation::run_impl (const double dt) { sw_bnd_flux_dif_k(icol,ilev,ibnd) = sw_bnd_flux_dn_k(icol,ilev,ibnd) - sw_bnd_flux_dir_k(icol,ilev,ibnd); }); // Compute surface fluxes - rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot_k, nswbands, sw_bnd_flux_dir_k, sw_bnd_flux_dif_k, sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), - std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1537,22 +1537,22 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); - real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); - real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); - real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably // does not matter in practice, as clouds probably should not be produced above 50 hPa and we // should not be encountering surface pressure above 1200 hPa, but in the event that things go off // the rails we might want to look at these still. - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), - std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Compute cloud-top diagnostics following AeroCOM recommendation @@ -1580,20 +1580,20 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // Get visible 0.67 micron band for COSP - auto idx_067_k = rrtmgp::get_wavelength_index_sw_k(0.67e-6); + auto idx_067_k = interface_t::get_wavelength_index_sw_k(0.67e-6); // Get IR 10.5 micron band for COSP - auto idx_105_k = rrtmgp::get_wavelength_index_lw_k(10.5e-6); - - real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - - rrtmgp::compute_aerocom_cloudtop( + auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); + + view_1d_real T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + view_1d_real eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + + interface_t::compute_aerocom_cloudtop( ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, @@ -1602,7 +1602,7 @@ void RRTMGPRadiation::run_impl (const double dt) { T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), - std::vector({ + std::vector({ T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); @@ -1699,7 +1699,7 @@ void RRTMGPRadiation::run_impl (const double dt) { }); #ifdef RRTMGP_ENABLE_YAKL // Sync back to gas_concs_k - real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + view_3d_real temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); Kokkos::deep_copy(temp, m_gas_concs_k.concs); #endif #endif @@ -1769,11 +1769,12 @@ void RRTMGPRadiation::run_impl (const double dt) { void RRTMGPRadiation::finalize_impl () { #ifdef RRTMGP_ENABLE_YAKL m_gas_concs.reset(); + rrtmgp::rrtmgp_finalize(); #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.reset(); + interface_t::rrtmgp_finalize(); #endif - rrtmgp::rrtmgp_finalize(); finalize_kls(); } diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 66c89476028b..28c5033c46d4 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -28,6 +28,10 @@ class RRTMGPRadiation : public AtmosphereProcess { template using uview_2d = Unmanaged>; + using layout_t = typename ekat::KokkosTypes::Layout; + + using interface_t = rrtmgp::rrtmgp_interface; + // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -92,7 +96,7 @@ class RRTMGPRadiation : public AtmosphereProcess { GasConcs m_gas_concs; #endif #ifdef RRTMGP_ENABLE_KOKKOS - GasConcsK m_gas_concs_k; + GasConcsK m_gas_concs_k; #endif // Prescribed greenhouse gas surface concentrations in moles / moles air @@ -137,15 +141,15 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + view_1d_real mu0_k; + view_1d_real sfc_alb_dir_vis_k; + view_1d_real sfc_alb_dir_nir_k; + view_1d_real sfc_alb_dif_vis_k; + view_1d_real sfc_alb_dif_nir_k; + view_1d_real sfc_flux_dir_vis_k; + view_1d_real sfc_flux_dir_nir_k; + view_1d_real sfc_flux_dif_vis_k; + view_1d_real sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) @@ -168,21 +172,21 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + view_2d_real p_lay_k; + view_2d_real t_lay_k; + view_2d_real z_del_k; + view_2d_real p_del_k; + view_2d_real qc_k; + view_2d_real nc_k; + view_2d_real qi_k; + view_2d_real cldfrac_tot_k; + view_2d_real eff_radius_qc_k; + view_2d_real eff_radius_qi_k; + view_2d_real tmp2d_k; + view_2d_real lwp_k; + view_2d_real iwp_k; + view_2d_real sw_heating_k; + view_2d_real lw_heating_k; #endif // 2d size (ncol, nlay+1) @@ -212,28 +216,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + view_2d_real p_lev_k; + view_2d_real t_lev_k; + view_2d_real sw_flux_up_k; + view_2d_real sw_flux_dn_k; + view_2d_real sw_flux_dn_dir_k; + view_2d_real lw_flux_up_k; + view_2d_real lw_flux_dn_k; + view_2d_real sw_clnclrsky_flux_up_k; + view_2d_real sw_clnclrsky_flux_dn_k; + view_2d_real sw_clnclrsky_flux_dn_dir_k; + view_2d_real sw_clrsky_flux_up_k; + view_2d_real sw_clrsky_flux_dn_k; + view_2d_real sw_clrsky_flux_dn_dir_k; + view_2d_real sw_clnsky_flux_up_k; + view_2d_real sw_clnsky_flux_dn_k; + view_2d_real sw_clnsky_flux_dn_dir_k; + view_2d_real lw_clnclrsky_flux_up_k; + view_2d_real lw_clnclrsky_flux_dn_k; + view_2d_real lw_clrsky_flux_up_k; + view_2d_real lw_clrsky_flux_dn_k; + view_2d_real lw_clnsky_flux_up_k; + view_2d_real lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -244,10 +248,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + view_3d_real sw_bnd_flux_up_k; + view_3d_real sw_bnd_flux_dn_k; + view_3d_real sw_bnd_flux_dir_k; + view_3d_real sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -256,8 +260,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + view_3d_real lw_bnd_flux_up_k; + view_3d_real lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -266,8 +270,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + view_2d_real sfc_alb_dir_k; + view_2d_real sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -278,10 +282,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + view_3d_real aero_tau_sw_k; + view_3d_real aero_ssa_sw_k; + view_3d_real aero_g_sw_k; + view_3d_real aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -290,8 +294,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + view_3d_real cld_tau_sw_bnd_k; + view_3d_real cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -300,8 +304,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + view_3d_real cld_tau_sw_gpt_k; + view_3d_real cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp index 54a32133da92..65cc99e09cc3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp @@ -1,25 +1,13 @@ -#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" #include "physics/rrtmgp/rrtmgp_test_utils.hpp" #ifdef RRTMGP_ENABLE_YAKL #include "YAKL_netcdf.h" #endif -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rte/mo_fluxes.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" - #include #include namespace rrtmgpTest { -#ifdef RRTMGP_ENABLE_YAKL -using yakl::fortran::parallel_for; -using yakl::fortran::SimpleBounds; -using yakl::intrinsics::mod; -using yakl::intrinsics::merge; -#endif - bool file_exists(const char *filename) { if (auto file = fopen(filename, "r")) { fclose(file); @@ -29,8 +17,12 @@ bool file_exists(const char *filename) { } } -// TODO: use YAKL intrinsics for this to avoid needing to make host copies #ifdef RRTMGP_ENABLE_YAKL +using yakl::fortran::parallel_for; +using yakl::fortran::SimpleBounds; +using yakl::intrinsics::mod; +using yakl::intrinsics::merge; + bool all_close(real2d &arr1, real2d &arr2, double tolerance) { int nx = arr1.dimension[0]; int ny = arr2.dimension[1]; @@ -46,28 +38,7 @@ bool all_close(real2d &arr1, real2d &arr2, double tolerance) { } return true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { - int nx = arr1.extent(0); - int ny = arr2.extent(1); - auto arr1_h = Kokkos::create_mirror_view(arr1); - auto arr2_h = Kokkos::create_mirror_view(arr2); - Kokkos::deep_copy(arr1_h, arr1); - Kokkos::deep_copy(arr2_h, arr2); - for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { - printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); - return false; - } - } - } - return true; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void dummy_atmos( std::string inputfile, int ncol, real2d &p_lay, real2d &t_lay, @@ -94,37 +65,7 @@ void dummy_atmos( // create a dummy atmosphere. dummy_clouds(scream::rrtmgp::cloud_optics_sw, p_lay, t_lay, lwp, iwp, rel, rei, cld); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_atmos( - std::string inputfile, - int ncol, real2dk &p_lay, real2dk &t_lay, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld) { - - // Setup boundary conditions, solar zenith angle, etc - // NOTE: this stuff would come from the model in a real run - // Ocean-ish values for surface albedos, just for example - Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); - - // Pick a solar zenith angle; this should come from the model - Kokkos::deep_copy(mu0, 0.86 ); - - // Get dummy cloud PHYSICAL properties. Note that this function call - // needs the CloudOptics object only because it uses the min and max - // valid values from the lookup tables for liquid and ice water path to - // create a dummy atmosphere. - dummy_clouds(scream::rrtmgp::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void dummy_clouds( CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cloud_mask) { @@ -151,38 +92,7 @@ void dummy_clouds( rei(icol,ilay) = merge(rei_val, 0._wp, iwp(icol,ilay) > 0._wp); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask) { - - // Problem sizes - int ncol = t_lay.extent(0); - int nlay = t_lay.extent(1); - - // Generate some fake liquid and ice water data. We pick values to be midway between - // the min and max of the valid lookup table values for effective radii - real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); - real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); - // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and - // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. - // Set sane values for liquid and ice water path. - // NOTE: these "sane" values are in g/m2! - Kokkos::parallel_for( conv::get_mdrp<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { - cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; - // Ice and liquid will overlap in a few layers - lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); - iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); - rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); - rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); - }); -} -#endif - -// Function to read fluxes from input file so we can compare our answers against the reference -#ifdef RRTMGP_ENABLE_YAKL void read_fluxes( std::string inputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -208,36 +118,7 @@ void read_fluxes( io.read(lw_flux_up, "lw_flux_up" ); io.read(lw_flux_dn, "lw_flux_dn" ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void read_fluxes( - std::string inputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - // Initialize netcdf reader - conv::SimpleNetCDF io; - io.open(inputfile, NC_NOWRITE); - - // Initialize arrays to hold fluxes - int nlev = io.getDimSize("lev"); - int ncol = io.getDimSize("col_flx"); - sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); - sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); - sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); - lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); - lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); - - // Read data - io.read(sw_flux_up, "sw_flux_up" ); - io.read(sw_flux_dn, "sw_flux_dn" ); - io.read(sw_flux_dir, "sw_flux_dir"); - io.read(lw_flux_up, "lw_flux_up" ); - io.read(lw_flux_dn, "lw_flux_dn" ); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void write_fluxes( std::string outputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -253,76 +134,5 @@ void write_fluxes( io.close(); } #endif -#ifdef RRTMGP_ENABLE_KOKKOS -void write_fluxes( - std::string outputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - conv::SimpleNetCDF io; - io.create(outputfile); - io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); - io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); - io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); - io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); - io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); - io.close(); -} -#endif - -// TODO: This function should instead take values, not file names, -// because for the test we do not want to write to file -#ifdef RRTMGP_ENABLE_YAKL -int compare_y(std::string file1, std::string file2) { - // Read data from baseline and test file - real2d sw_flux_up_1; - real2d sw_flux_dn_1; - real2d sw_flux_dir_1; - real2d lw_flux_up_1; - real2d lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2d sw_flux_up_2; - real2d sw_flux_dn_2; - real2d sw_flux_dir_2; - real2d lw_flux_up_2; - real2d lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int compare_k(std::string file1, std::string file2) { - // Read data from baseline and test file - real2dk sw_flux_up_1; - real2dk sw_flux_dn_1; - real2dk sw_flux_dir_1; - real2dk lw_flux_up_1; - real2dk lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2dk sw_flux_up_2; - real2dk sw_flux_dn_2; - real2dk sw_flux_dir_2; - real2dk lw_flux_up_2; - real2dk lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index b9498a9a34f9..79f6e570950d 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -1,5 +1,10 @@ #ifndef RRTMGP_TEST_UTILS_HPP #define RRTMGP_TEST_UTILS_HPP + +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rte/mo_fluxes.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" namespace rrtmgpTest { @@ -37,33 +42,134 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance); +template +struct rrtmgp_test_utils { -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +using real1dk = typename ekat::KokkosTypes::template view_1d; +using real2dk = typename ekat::KokkosTypes::template view_2d; +using real3dk = typename ekat::KokkosTypes::template view_3d; +using MDRP = typename conv::MDRP; +using interface_t = scream::rrtmgp::rrtmgp_interface; -void dummy_atmos( +static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) +{ + int nx = arr1.extent(0); + int ny = arr2.extent(1); + auto arr1_h = Kokkos::create_mirror_view(arr1); + auto arr2_h = Kokkos::create_mirror_view(arr2); + Kokkos::deep_copy(arr1_h, arr1); + Kokkos::deep_copy(arr2_h, arr2); + for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { + printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); + return false; + } + } + } + return true; +} + +static void dummy_clouds( + CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask +) +{ + // Problem sizes + int ncol = t_lay.extent(0); + int nlay = t_lay.extent(1); + + // Generate some fake liquid and ice water data. We pick values to be midway between + // the min and max of the valid lookup table values for effective radii + real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); + real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); + + // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and + // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. + // Set sane values for liquid and ice water path. + // NOTE: these "sane" values are in g/m2! + Kokkos::parallel_for( MDRP::template get<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { + cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; + // Ice and liquid will overlap in a few layers + lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); + iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); + rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); + rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); + }); +} + +static void dummy_atmos( std::string inputfile, int ncol, real2dk &p_lay, real2dk &t_lay, real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, real1dk &mu0, real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +) +{ + // Setup boundary conditions, solar zenith angle, etc + // NOTE: this stuff would come from the model in a real run -void read_fluxes( + // Ocean-ish values for surface albedos, just for example + Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); + + // Pick a solar zenith angle; this should come from the model + Kokkos::deep_copy(mu0, 0.86 ); + + // Get dummy cloud PHYSICAL properties. Note that this function call + // needs the CloudOptics object only because it uses the min and max + // valid values from the lookup tables for liquid and ice water path to + // create a dummy atmosphere. + dummy_clouds(interface_t::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); +} + +static void read_fluxes( std::string inputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + // Initialize netcdf reader + conv::SimpleNetCDF io; + io.open(inputfile, NC_NOWRITE); -void write_fluxes( + // Initialize arrays to hold fluxes + int nlev = io.getDimSize("lev"); + int ncol = io.getDimSize("col_flx"); + sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); + sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); + sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); + lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); + lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); + + // Read data + io.read(sw_flux_up, "sw_flux_up" ); + io.read(sw_flux_dn, "sw_flux_dn" ); + io.read(sw_flux_dir, "sw_flux_dir"); + io.read(lw_flux_up, "lw_flux_up" ); + io.read(lw_flux_dn, "lw_flux_dn" ); +} + +static void write_fluxes( std::string outputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + conv::SimpleNetCDF io; + io.create(outputfile); + io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); + io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); + io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); + io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); + io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); + io.close(); +} + +}; #endif } diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 119093d5f9d3..6e924103229f 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -109,8 +109,9 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, RealT xmin, RealT xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 1696e7933685..10281df8078c 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,11 +1,4 @@ #include "scream_rrtmgp_interface.hpp" -#include "cpp/examples/mo_load_coefficients.h" -#include "examples/all-sky/mo_load_cloud_coefficients.h" -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" -#include "cpp/rte/mo_rte_sw.h" -#include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" namespace scream { diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index c40be4904302..2038cb62ff79 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -4,7 +4,14 @@ #include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" +#include "cpp/examples/mo_load_coefficients.h" #include "cpp/rrtmgp_const.h" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "cpp/rte/mo_rte_sw.h" +#include "cpp/rte/mo_rte_lw.h" +#include "examples/all-sky/mo_load_cloud_coefficients.h" #include "rrtmgp_utils.hpp" @@ -152,7 +159,7 @@ int get_wavelength_index_lw(double wavelength); // New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_interface { using MDRP = typename conv::MDRP; @@ -469,8 +476,8 @@ static void rrtmgp_main( #endif // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); @@ -640,7 +647,7 @@ static void rrtmgp_sw( // Subset gases auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; + GasConcsK gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { auto vmr_day = view_t("vmr_day", nday, nlay); @@ -653,7 +660,7 @@ static void rrtmgp_sw( } // Subset aerosol optics - OpticalProps2strK aerosol_day; + OpticalProps2strK aerosol_day; aerosol_day.init(k_dist.get_band_lims_wavenumber()); aerosol_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { @@ -664,7 +671,7 @@ static void rrtmgp_sw( // Subset cloud optics // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; + OpticalProps2strK clouds_day; clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); clouds_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { @@ -690,7 +697,7 @@ static void rrtmgp_sw( auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; + FluxesBybandK fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; fluxes_day.flux_dn_dir = flux_dn_dir_day; @@ -699,10 +706,10 @@ static void rrtmgp_sw( fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; // Allocate space for optical properties - OpticalProps2strK optics; + OpticalProps2strK optics; optics.alloc_2str(nday, nlay, k_dist); - OpticalProps2strK optics_no_aerosols; + OpticalProps2strK optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_2str(nday, nlay, k_dist); @@ -1195,8 +1202,8 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template -static void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { +template +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); @@ -1291,7 +1298,7 @@ static OpticalProps1sclK get_cloud_optics_lw( } template -OpticalProps2strK get_subsampled_clouds( +static OpticalProps2strK get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { // Initialized subsampled optics @@ -1342,7 +1349,7 @@ OpticalProps2strK get_subsampled_clouds( } template -OpticalProps1sclK get_subsampled_clouds( +static OpticalProps1sclK get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index ddef61808fe0..36e2a59e02ff 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -849,6 +849,14 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { #endif #ifdef RRTMGP_ENABLE_KOKKOS +using interface_t = scream::rrtmgp::rrtmgp_interface<>; +using real1dk = interface_t::view_t; +using real2dk = interface_t::view_t; +using real3dk = interface_t::view_t; +using int1dk = interface_t::view_t; +using int2dk = interface_t::view_t; +using int3dk = interface_t::view_t; + TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos scream::init_kls(); @@ -921,7 +929,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 1.0; }); auto cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with no cloud @@ -931,7 +939,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.0; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with empty clouds (cloud fraction but with no associated mixing ratio) @@ -943,7 +951,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.1; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with cell half filled with cloud @@ -953,7 +961,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.5; }); cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up @@ -977,14 +985,14 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { }); // Limit to bounds that contain the data; should be no change in values - scream::rrtmgp::limit_to_bounds(arr, 0.0, 5.0, arr_limited); + interface_t::limit_to_bounds_k(arr, 0.0, 5.0, arr_limited); REQUIRE(chc(arr)(0,0) == chc(arr_limited)(0,0)); REQUIRE(chc(arr)(0,1) == chc(arr_limited)(0,1)); REQUIRE(chc(arr)(1,0) == chc(arr_limited)(1,0)); REQUIRE(chc(arr)(1,1) == chc(arr_limited)(1,1)); // Limit to bounds that do not completely contain the data; should be a change in values! - scream::rrtmgp::limit_to_bounds(arr, 1.5, 3.5, arr_limited); + interface_t::limit_to_bounds_k(arr, 1.5, 3.5, arr_limited); REQUIRE(chc(arr_limited)(0,0) == 1.5); REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); @@ -1078,7 +1086,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Create simple test cases; We expect, given the input data, that band 10 // will straddle the NIR and VIS, bands 1-9 will be purely NIR, and bands 11-14 @@ -1106,7 +1114,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1138,7 +1146,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1169,7 +1177,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1200,7 +1208,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1216,7 +1224,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Finalize YAKL logger->info("Free memory...\n"); - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); gas_concs.reset(); scream::finalize_kls(); } @@ -1282,7 +1290,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { @@ -1316,7 +1324,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); auto cldmask_h = chc(cldmask); for (int igpt = 0; igpt < ngpt; igpt++) { if (cldmask_h(0,0,igpt) == 1) { @@ -1359,7 +1367,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); // Case: @@ -1376,7 +1384,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 1; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); // Case: @@ -1393,11 +1401,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1.0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0 / 3.0); // Case: @@ -1414,11 +1422,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); scream::finalize_kls(); } @@ -1463,7 +1471,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { Kokkos::deep_copy(rel, 10.0); Kokkos::deep_copy(rei, 10.0); // Call the function - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1481,7 +1489,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { // Case 2: if all clouds, everything goes to 1 * its value Kokkos::deep_copy(cldfrac_tot, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1504,7 +1512,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 3) = 0.3; cldfrac_tot(0, 4) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1520,7 +1528,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 5) = 0.4; cldfrac_tot(0, 6) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1534,7 +1542,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 4) = 0.0; cldfrac_tot(0, 5) = 0.1; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1550,7 +1558,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 1.0); Kokkos::deep_copy(qi, 0.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1568,7 +1576,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 0.0); Kokkos::deep_copy(qi, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1604,7 +1612,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { qc(0, 6) = 50; qc(0, 7) = 10; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, From 5c8349144ea47bbc4e27bc2bdd2b96227689173f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 14:15:42 -0600 Subject: [PATCH 249/477] Everything working --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 327 +++++++++--------- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 158 ++++----- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 10 +- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 89 ++++- .../rrtmgp/scream_rrtmgp_interface.hpp | 24 +- .../src/physics/rrtmgp/tests/rrtmgp_tests.cpp | 64 ++-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 15 +- .../rrtmgp/rrtmgp_standalone_unit.cpp | 37 +- 9 files changed, 405 insertions(+), 321 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 3d7fba5ac72d..4c542032b3ac 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 3d7fba5ac72dab7c8448411e7452e2b409f22d93 +Subproject commit 4c542032b3ac3f842a9619c9eadc005c507a0366 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index a24f4d5ddc32..1556280eab4b 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -19,6 +19,28 @@ namespace scream { +template +void check_views2(const View1& view1, const View2& view2) +{ + constexpr auto krank1 = View1::rank; + constexpr auto krank2 = View2::rank; + + RRT_REQUIRE(krank1 == krank2, "Rank mismatch for: " << view1.label()); + RRT_REQUIRE(krank1 == 2, "Rank mismatch for: " << view1.label()); + + for (auto r = 0; r < krank1; ++r) { + RRT_REQUIRE(view1.extent(r) == view2.extent(r), "Dim mismatch for: " << view1.label() << ", rank: " << r << ", " << view1.extent(r) << " != " << view2.extent(r)); + } + + for (auto i = 0; i < view1.extent(0); ++i) { + for (auto j = 0; j < view1.extent(1); ++j) { + const auto data1 = view1(i, j); + const auto data2 = view2(i, j); + RRT_REQUIRE(data1 == data2, "Data mismatch for: " << view1.label() << ", i: " << i << ", j: " << j << ", " << data1 << " != " << data2); + } + } +} + using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; @@ -596,7 +618,7 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // Names of active gases auto gas_names_yakl_offset = string1dv(m_ngas); - m_gas_mol_weights = view_1d_real("gas_mol_weights",m_ngas); + m_gas_mol_weights = real1dk("gas_mol_weights",m_ngas); // the lookup function for getting the gas mol weights doesn't work on device auto gas_mol_w_host = Kokkos::create_mirror_view(m_gas_mol_weights); for (int igas = 0; igas < m_ngas; igas++) { @@ -848,11 +870,9 @@ void RRTMGPRadiation::run_impl (const double dt) { "[RRTMGP::run_impl] Col chunk beg,end: " + std::to_string(beg) + ", " + std::to_string(beg+ncol) + "\n"); +#ifdef RRTMGP_ENABLE_YAKL // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. - // JGF: this doesn't appear to be copying the data, just returning a new array - // pointing to the same memory. -#ifdef RRTMGP_ENABLE_YAKL auto subview_1d = [&](const real1d v) -> real1d { return real1d(v.label(),v.myData,ncol); }; @@ -923,31 +943,64 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // If YAKL is on, we don't want aliased memory in both the yakl and kokos - // subviews. - auto subview_1dk = [&](const view_1d_real v) -> view_1d_real { - view_1d_real subv(v, std::make_pair(0, ncol)); + // subviews, so make new views and deep_copy. Also, be sure to trim the excess + // items from the field manager views due to simd packs. If we don't trim, then + // check_range_k will fail due to looking at unused values. Once rrtmgp can handle + // packs, this won't be necessary. + auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { + ureal1dk subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - view_1d_real rv(v.label(), ncol); + real1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_2dk = [&](const view_2d_real v) -> view_2d_real { - view_2d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { + cureal1dk subv(v, std::make_pair(0, ncol)); #ifdef RRTMGP_ENABLE_YAKL - view_2d_real rv(v.label(), ncol, v.extent(1)); + creal1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); return rv; #else return subv; #endif }; - auto subview_3dk = [&](const view_3d_real v) -> view_3d_real { - view_3d_real subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + auto subview_2dk = [&](const ureal2dk& v) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); #ifdef RRTMGP_ENABLE_YAKL - view_3d_real rv(v.label(), ncol, v.extent(1), v.extent(2)); + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_2dkc = [&](const cureal2dk& v, const int inner_dim) -> cureal2dk { + cureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); +#ifdef RRTMGP_ENABLE_YAKL + creal2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_2dk_new = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); +#ifdef RRTMGP_ENABLE_YAKL + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); + return rv; +#else + return subv; +#endif + }; + auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { + ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); +#ifdef RRTMGP_ENABLE_YAKL + real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); return rv; #else @@ -955,59 +1008,59 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; - auto p_lay_k = subview_2dk(m_buffer.p_lay_k); - auto t_lay_k = subview_2dk(m_buffer.t_lay_k); - auto p_lev_k = subview_2dk(m_buffer.p_lev_k); - auto z_del_k = subview_2dk(m_buffer.z_del_k); - auto p_del_k = subview_2dk(m_buffer.p_del_k); - auto t_lev_k = subview_2dk(m_buffer.t_lev_k); - auto mu0_k = subview_1dk(m_buffer.mu0_k); - auto sfc_alb_dir_k = subview_2dk(m_buffer.sfc_alb_dir_k); - auto sfc_alb_dif_k = subview_2dk(m_buffer.sfc_alb_dif_k); - auto sfc_alb_dir_vis_k = subview_1dk(m_buffer.sfc_alb_dir_vis_k); - auto sfc_alb_dir_nir_k = subview_1dk(m_buffer.sfc_alb_dir_nir_k); - auto sfc_alb_dif_vis_k = subview_1dk(m_buffer.sfc_alb_dif_vis_k); - auto sfc_alb_dif_nir_k = subview_1dk(m_buffer.sfc_alb_dif_nir_k); - auto qc_k = subview_2dk(m_buffer.qc_k); - auto nc_k = subview_2dk(m_buffer.nc_k); - auto qi_k = subview_2dk(m_buffer.qi_k); - auto cldfrac_tot_k = subview_2dk(m_buffer.cldfrac_tot_k); - auto rel_k = subview_2dk(m_buffer.eff_radius_qc_k); - auto rei_k = subview_2dk(m_buffer.eff_radius_qi_k); - auto sw_flux_up_k = subview_2dk(m_buffer.sw_flux_up_k); - auto sw_flux_dn_k = subview_2dk(m_buffer.sw_flux_dn_k); - auto sw_flux_dn_dir_k = subview_2dk(m_buffer.sw_flux_dn_dir_k); - auto lw_flux_up_k = subview_2dk(m_buffer.lw_flux_up_k); - auto lw_flux_dn_k = subview_2dk(m_buffer.lw_flux_dn_k); - auto sw_clnclrsky_flux_up_k = subview_2dk(m_buffer.sw_clnclrsky_flux_up_k); - auto sw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_k); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_dir_k); - auto sw_clrsky_flux_up_k = subview_2dk(m_buffer.sw_clrsky_flux_up_k); - auto sw_clrsky_flux_dn_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_k); - auto sw_clrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_dir_k); - auto sw_clnsky_flux_up_k = subview_2dk(m_buffer.sw_clnsky_flux_up_k); - auto sw_clnsky_flux_dn_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_k); - auto sw_clnsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_dir_k); - auto lw_clnclrsky_flux_up_k = subview_2dk(m_buffer.lw_clnclrsky_flux_up_k); - auto lw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.lw_clnclrsky_flux_dn_k); - auto lw_clrsky_flux_up_k = subview_2dk(m_buffer.lw_clrsky_flux_up_k); - auto lw_clrsky_flux_dn_k = subview_2dk(m_buffer.lw_clrsky_flux_dn_k); - auto lw_clnsky_flux_up_k = subview_2dk(m_buffer.lw_clnsky_flux_up_k); - auto lw_clnsky_flux_dn_k = subview_2dk(m_buffer.lw_clnsky_flux_dn_k); - auto sw_bnd_flux_up_k = subview_3dk(m_buffer.sw_bnd_flux_up_k); - auto sw_bnd_flux_dn_k = subview_3dk(m_buffer.sw_bnd_flux_dn_k); - auto sw_bnd_flux_dir_k = subview_3dk(m_buffer.sw_bnd_flux_dir_k); - auto sw_bnd_flux_dif_k = subview_3dk(m_buffer.sw_bnd_flux_dif_k); - auto lw_bnd_flux_up_k = subview_3dk(m_buffer.lw_bnd_flux_up_k); - auto lw_bnd_flux_dn_k = subview_3dk(m_buffer.lw_bnd_flux_dn_k); - auto sfc_flux_dir_vis_k = subview_1dk(m_buffer.sfc_flux_dir_vis_k); - auto sfc_flux_dir_nir_k = subview_1dk(m_buffer.sfc_flux_dir_nir_k); - auto sfc_flux_dif_vis_k = subview_1dk(m_buffer.sfc_flux_dif_vis_k); - auto sfc_flux_dif_nir_k = subview_1dk(m_buffer.sfc_flux_dif_nir_k); - auto aero_tau_sw_k = subview_3dk(m_buffer.aero_tau_sw_k); - auto aero_ssa_sw_k = subview_3dk(m_buffer.aero_ssa_sw_k); - auto aero_g_sw_k = subview_3dk(m_buffer.aero_g_sw_k); - auto aero_tau_lw_k = subview_3dk(m_buffer.aero_tau_lw_k); + // Note, ncol will not necessary be m_col_chunk_size because the number of cols + // will not always be evenly divided by m_col_chunk_size. In most cases, the + // extra space will not cause any problems, but it does sometimes. + auto p_lay_k = subview_2dkc(d_pmid, m_nlay); + auto t_lay_k = subview_2dkc(d_tmid, m_nlay); + auto p_lev_k = subview_2dkc(d_pint, m_nlay+1); + auto p_del_k = subview_2dkc(d_pdel, m_nlay); + auto sfc_alb_dir_k = m_buffer.sfc_alb_dir_k; + auto sfc_alb_dif_k = m_buffer.sfc_alb_dif_k; + auto sfc_alb_dir_vis_k = subview_1dkc(d_sfc_alb_dir_vis); + auto sfc_alb_dir_nir_k = subview_1dkc(d_sfc_alb_dir_nir); + auto sfc_alb_dif_vis_k = subview_1dkc(d_sfc_alb_dif_vis); + auto sfc_alb_dif_nir_k = subview_1dkc(d_sfc_alb_dif_nir); + auto qc_k = subview_2dkc(d_qc, m_nlay); + auto nc_k = subview_2dkc(d_nc, m_nlay); + auto qi_k = subview_2dkc(d_qi, m_nlay); + auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; + auto rel_k = subview_2dkc(d_rel, m_nlay); + auto rei_k = subview_2dkc(d_rei, m_nlay); + auto sw_flux_up_k = subview_2dk_new(d_sw_flux_up, m_nlay+1); + auto sw_flux_dn_k = subview_2dk_new(d_sw_flux_dn, m_nlay+1); + auto sw_flux_dn_dir_k = subview_2dk_new(d_sw_flux_dn_dir, m_nlay+1); + auto lw_flux_up_k = subview_2dk_new(d_lw_flux_up, m_nlay+1); + auto lw_flux_dn_k = subview_2dk_new(d_lw_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_up_k = subview_2dk_new(d_sw_clnclrsky_flux_up, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = subview_2dk_new(d_sw_clnclrsky_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); + auto sw_clrsky_flux_up_k = subview_2dk_new(d_sw_clrsky_flux_up, m_nlay+1); + auto sw_clrsky_flux_dn_k = subview_2dk_new(d_sw_clrsky_flux_dn, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clrsky_flux_dn_dir, m_nlay+1); + auto sw_clnsky_flux_up_k = subview_2dk_new(d_sw_clnsky_flux_up, m_nlay+1); + auto sw_clnsky_flux_dn_k = subview_2dk_new(d_sw_clnsky_flux_dn, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnsky_flux_dn_dir, m_nlay+1); + auto lw_clnclrsky_flux_up_k = subview_2dk_new(d_lw_clnclrsky_flux_up, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = subview_2dk_new(d_lw_clnclrsky_flux_dn, m_nlay+1); + auto lw_clrsky_flux_up_k = subview_2dk_new(d_lw_clrsky_flux_up, m_nlay+1); + auto lw_clrsky_flux_dn_k = subview_2dk_new(d_lw_clrsky_flux_dn, m_nlay+1); + auto lw_clnsky_flux_up_k = subview_2dk_new(d_lw_clnsky_flux_up, m_nlay+1); + auto lw_clnsky_flux_dn_k = subview_2dk_new(d_lw_clnsky_flux_dn, m_nlay+1); + auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; + auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; + auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; + auto sw_bnd_flux_dif_k = m_buffer.sw_bnd_flux_dif_k; + auto lw_bnd_flux_up_k = m_buffer.lw_bnd_flux_up_k; + auto lw_bnd_flux_dn_k = m_buffer.lw_bnd_flux_dn_k; + auto sfc_flux_dir_vis_k = subview_1dk(d_sfc_flux_dir_vis); + auto sfc_flux_dir_nir_k = subview_1dk(d_sfc_flux_dir_nir); + auto sfc_flux_dif_vis_k = subview_1dk(d_sfc_flux_dif_vis); + auto sfc_flux_dif_nir_k = subview_1dk(d_sfc_flux_dif_nir); + auto aero_tau_sw_k = m_buffer.aero_tau_sw_k; + auto aero_ssa_sw_k = m_buffer.aero_ssa_sw_k; + auto aero_g_sw_k = m_buffer.aero_g_sw_k; + auto aero_tau_lw_k = m_buffer.aero_tau_lw_k; auto cld_tau_sw_bnd_k = subview_3dk(m_buffer.cld_tau_sw_bnd_k); auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); @@ -1015,7 +1068,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif auto d_tint = m_buffer.d_tint; auto d_dz = m_buffer.d_dz; - + auto d_mu0 = m_buffer.cosine_zenith; // Set gas concs to "view" only the first ncol columns #ifdef RRTMGP_ENABLE_YAKL @@ -1032,7 +1085,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // Determine the cosine zenith angle // NOTE: Since we are bridging to F90 arrays this must be done on HOST and then // deep copied to a device view. - auto d_mu0 = m_buffer.cosine_zenith; auto h_mu0 = Kokkos::create_mirror_view(d_mu0); if (m_fixed_solar_zenith_angle > 0) { for (int i=0; i({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1206,7 +1226,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real2d tmp2d = subview_2d(m_buffer.tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real tmp2d_k = subview_2dk(m_buffer.tmp2d_k); + real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1315,7 +1335,7 @@ void RRTMGPRadiation::run_impl (const double dt) { interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), - std::vector({lwp_k, iwp_k})); + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1356,7 +1376,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), - std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1386,9 +1406,9 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS interface_t::rrtmgp_main( ncol, m_nlay, - p_lay_k, t_lay_k, p_lev_k, t_lev_k, + p_lay_k, t_lay_k, p_lev_k, d_tint, m_gas_concs_k, - sfc_alb_dir_k, sfc_alb_dif_k, mu0_k, + sfc_alb_dir_k, sfc_alb_dif_k, d_mu0, lwp_k, iwp_k, rel_k, rei_k, cldfrac_tot_k, aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k, cld_tau_sw_bnd_k, cld_tau_lw_bnd_k, @@ -1412,7 +1432,7 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, lw_clrsky_flux_up, lw_clrsky_flux_dn, lw_clnsky_flux_up, lw_clnsky_flux_dn}), - std::vector({ + std::vector({ sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, @@ -1420,9 +1440,8 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up_k, lw_clnclrsky_flux_dn_k, lw_clrsky_flux_up_k, lw_clrsky_flux_dn_k, lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); - COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), - std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1472,7 +1491,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), - std::vector({sw_heating_k, lw_heating_k})); + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1516,7 +1535,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), - std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1537,10 +1556,10 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_1d_real cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the // exception that I removed the restriction on low clouds to be above (numerically lower pressures) // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably @@ -1552,7 +1571,7 @@ void RRTMGPRadiation::run_impl (const double dt) { interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), - std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Compute cloud-top diagnostics following AeroCOM recommendation @@ -1584,17 +1603,17 @@ void RRTMGPRadiation::run_impl (const double dt) { // Get IR 10.5 micron band for COSP auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); - view_1d_real T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - view_1d_real eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); interface_t::compute_aerocom_cloudtop( - ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, + ncol, nlay, t_lay_k, p_lay_k, p_del_k, d_dz, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k); @@ -1602,7 +1621,7 @@ void RRTMGPRadiation::run_impl (const double dt) { T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), - std::vector({ + std::vector({ T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); @@ -1658,34 +1677,8 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; - d_sfc_flux_dir_nir(icol) = sfc_flux_dir_nir_k(i); - d_sfc_flux_dir_vis(icol) = sfc_flux_dir_vis_k(i); - d_sfc_flux_dif_nir(icol) = sfc_flux_dif_nir_k(i); - d_sfc_flux_dif_vis(icol) = sfc_flux_dif_vis_k(i); d_sfc_flux_sw_net(icol) = sw_flux_dn_k(i,kbot_k) - sw_flux_up_k(i,kbot_k); d_sfc_flux_lw_dn(icol) = lw_flux_dn_k(i,kbot_k); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - d_sw_flux_up(icol,k) = sw_flux_up_k(i,k); - d_sw_flux_dn(icol,k) = sw_flux_dn_k(i,k); - d_sw_flux_dn_dir(icol,k) = sw_flux_dn_dir_k(i,k); - d_lw_flux_up(icol,k) = lw_flux_up_k(i,k); - d_lw_flux_dn(icol,k) = lw_flux_dn_k(i,k); - d_sw_clnclrsky_flux_up(icol,k) = sw_clnclrsky_flux_up_k(i,k); - d_sw_clnclrsky_flux_dn(icol,k) = sw_clnclrsky_flux_dn_k(i,k); - d_sw_clnclrsky_flux_dn_dir(icol,k) = sw_clnclrsky_flux_dn_dir_k(i,k); - d_sw_clrsky_flux_up(icol,k) = sw_clrsky_flux_up_k(i,k); - d_sw_clrsky_flux_dn(icol,k) = sw_clrsky_flux_dn_k(i,k); - d_sw_clrsky_flux_dn_dir(icol,k) = sw_clrsky_flux_dn_dir_k(i,k); - d_sw_clnsky_flux_up(icol,k) = sw_clnsky_flux_up_k(i,k); - d_sw_clnsky_flux_dn(icol,k) = sw_clnsky_flux_dn_k(i,k); - d_sw_clnsky_flux_dn_dir(icol,k) = sw_clnsky_flux_dn_dir_k(i,k); - d_lw_clnclrsky_flux_up(icol,k) = lw_clnclrsky_flux_up_k(i,k); - d_lw_clnclrsky_flux_dn(icol,k) = lw_clnclrsky_flux_dn_k(i,k); - d_lw_clrsky_flux_up(icol,k) = lw_clrsky_flux_up_k(i,k); - d_lw_clrsky_flux_dn(icol,k) = lw_clrsky_flux_dn_k(i,k); - d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up_k(i,k); - d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn_k(i,k); - }); // Extract optical properties for COSP Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067_k); @@ -1699,7 +1692,7 @@ void RRTMGPRadiation::run_impl (const double dt) { }); #ifdef RRTMGP_ENABLE_YAKL // Sync back to gas_concs_k - view_3d_real temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); Kokkos::deep_copy(temp, m_gas_concs_k.concs); #endif #endif diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 28c5033c46d4..c60a1cb3181a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -16,17 +16,21 @@ namespace scream { class RRTMGPRadiation : public AtmosphereProcess { public: - using view_1d_real = typename ekat::KokkosTypes::template view_1d; - using view_2d_real = typename ekat::KokkosTypes::template view_2d; - using view_3d_real = typename ekat::KokkosTypes::template view_3d; - using view_2d_real_const = typename ekat::KokkosTypes::template view_2d; - using ci_string = ekat::CaseInsensitiveString; - - using KT = ekat::KokkosTypes; - template - using uview_1d = Unmanaged>; - template - using uview_2d = Unmanaged>; + using KT = ekat::KokkosTypes; + using real1dk = typename KT::template view_1d; + using real2dk = typename KT::template view_2d; + using real3dk = typename KT::template view_3d; + using creal1dk = typename KT::template view_1d; + using creal2dk = typename KT::template view_2d; + using creal3dk = typename KT::template view_3d; + using ureal1dk = Unmanaged; + using ureal2dk = Unmanaged; + using ureal3dk = Unmanaged; + using cureal1dk = Unmanaged; + using cureal2dk = Unmanaged; + using cureal3dk = Unmanaged; + + using ci_string = ekat::CaseInsensitiveString; using layout_t = typename ekat::KokkosTypes::Layout; @@ -91,7 +95,7 @@ class RRTMGPRadiation : public AtmosphereProcess { // These are the gases that we keep track of int m_ngas; std::vector m_gas_names; - view_1d_real m_gas_mol_weights; + real1dk m_gas_mol_weights; #ifdef RRTMGP_ENABLE_YAKL GasConcs m_gas_concs; #endif @@ -128,7 +132,7 @@ class RRTMGPRadiation : public AtmosphereProcess { static constexpr int num_3d_nlay_nlwgpts = 1; // 1d size (ncol) - uview_1d cosine_zenith; + ureal1dk cosine_zenith; #ifdef RRTMGP_ENABLE_YAKL real1d mu0; real1d sfc_alb_dir_vis; @@ -141,19 +145,19 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_1d_real mu0_k; - view_1d_real sfc_alb_dir_vis_k; - view_1d_real sfc_alb_dir_nir_k; - view_1d_real sfc_alb_dif_vis_k; - view_1d_real sfc_alb_dif_nir_k; - view_1d_real sfc_flux_dir_vis_k; - view_1d_real sfc_flux_dir_nir_k; - view_1d_real sfc_flux_dif_vis_k; - view_1d_real sfc_flux_dif_nir_k; + real1dk mu0_k; + real1dk sfc_alb_dir_vis_k; + real1dk sfc_alb_dir_nir_k; + real1dk sfc_alb_dif_vis_k; + real1dk sfc_alb_dif_nir_k; + real1dk sfc_flux_dir_vis_k; + real1dk sfc_flux_dir_nir_k; + real1dk sfc_flux_dif_vis_k; + real1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) - uview_2d d_dz; + ureal2dk d_dz; #ifdef RRTMGP_ENABLE_YAKL real2d p_lay; real2d t_lay; @@ -172,25 +176,25 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real p_lay_k; - view_2d_real t_lay_k; - view_2d_real z_del_k; - view_2d_real p_del_k; - view_2d_real qc_k; - view_2d_real nc_k; - view_2d_real qi_k; - view_2d_real cldfrac_tot_k; - view_2d_real eff_radius_qc_k; - view_2d_real eff_radius_qi_k; - view_2d_real tmp2d_k; - view_2d_real lwp_k; - view_2d_real iwp_k; - view_2d_real sw_heating_k; - view_2d_real lw_heating_k; + real2dk p_lay_k; + real2dk t_lay_k; + real2dk z_del_k; + real2dk p_del_k; + real2dk qc_k; + real2dk nc_k; + real2dk qi_k; + real2dk cldfrac_tot_k; + real2dk eff_radius_qc_k; + real2dk eff_radius_qi_k; + real2dk tmp2d_k; + real2dk lwp_k; + real2dk iwp_k; + real2dk sw_heating_k; + real2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) - uview_2d d_tint; + ureal2dk d_tint; #ifdef RRTMGP_ENABLE_YAKL real2d p_lev; real2d t_lev; @@ -216,28 +220,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real p_lev_k; - view_2d_real t_lev_k; - view_2d_real sw_flux_up_k; - view_2d_real sw_flux_dn_k; - view_2d_real sw_flux_dn_dir_k; - view_2d_real lw_flux_up_k; - view_2d_real lw_flux_dn_k; - view_2d_real sw_clnclrsky_flux_up_k; - view_2d_real sw_clnclrsky_flux_dn_k; - view_2d_real sw_clnclrsky_flux_dn_dir_k; - view_2d_real sw_clrsky_flux_up_k; - view_2d_real sw_clrsky_flux_dn_k; - view_2d_real sw_clrsky_flux_dn_dir_k; - view_2d_real sw_clnsky_flux_up_k; - view_2d_real sw_clnsky_flux_dn_k; - view_2d_real sw_clnsky_flux_dn_dir_k; - view_2d_real lw_clnclrsky_flux_up_k; - view_2d_real lw_clnclrsky_flux_dn_k; - view_2d_real lw_clrsky_flux_up_k; - view_2d_real lw_clrsky_flux_dn_k; - view_2d_real lw_clnsky_flux_up_k; - view_2d_real lw_clnsky_flux_dn_k; + real2dk p_lev_k; + real2dk t_lev_k; + real2dk sw_flux_up_k; + real2dk sw_flux_dn_k; + real2dk sw_flux_dn_dir_k; + real2dk lw_flux_up_k; + real2dk lw_flux_dn_k; + real2dk sw_clnclrsky_flux_up_k; + real2dk sw_clnclrsky_flux_dn_k; + real2dk sw_clnclrsky_flux_dn_dir_k; + real2dk sw_clrsky_flux_up_k; + real2dk sw_clrsky_flux_dn_k; + real2dk sw_clrsky_flux_dn_dir_k; + real2dk sw_clnsky_flux_up_k; + real2dk sw_clnsky_flux_dn_k; + real2dk sw_clnsky_flux_dn_dir_k; + real2dk lw_clnclrsky_flux_up_k; + real2dk lw_clnclrsky_flux_dn_k; + real2dk lw_clrsky_flux_up_k; + real2dk lw_clrsky_flux_dn_k; + real2dk lw_clnsky_flux_up_k; + real2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -248,10 +252,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real sw_bnd_flux_up_k; - view_3d_real sw_bnd_flux_dn_k; - view_3d_real sw_bnd_flux_dir_k; - view_3d_real sw_bnd_flux_dif_k; + real3dk sw_bnd_flux_up_k; + real3dk sw_bnd_flux_dn_k; + real3dk sw_bnd_flux_dir_k; + real3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -260,8 +264,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real lw_bnd_flux_up_k; - view_3d_real lw_bnd_flux_dn_k; + real3dk lw_bnd_flux_up_k; + real3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -270,8 +274,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_2d_real sfc_alb_dir_k; - view_2d_real sfc_alb_dif_k; + real2dk sfc_alb_dir_k; + real2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -282,10 +286,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real aero_tau_sw_k; - view_3d_real aero_ssa_sw_k; - view_3d_real aero_g_sw_k; - view_3d_real aero_tau_lw_k; + real3dk aero_tau_sw_k; + real3dk aero_ssa_sw_k; + real3dk aero_g_sw_k; + real3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -294,8 +298,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real cld_tau_sw_bnd_k; - view_3d_real cld_tau_lw_bnd_k; + real3dk cld_tau_sw_bnd_k; + real3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -304,8 +308,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - view_3d_real cld_tau_sw_gpt_k; - view_3d_real cld_tau_lw_gpt_k; + real3dk cld_tau_sw_gpt_k; + real3dk cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index 79f6e570950d..466467919fcf 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -42,14 +42,14 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_test_utils { -using real1dk = typename ekat::KokkosTypes::template view_1d; -using real2dk = typename ekat::KokkosTypes::template view_2d; -using real3dk = typename ekat::KokkosTypes::template view_3d; -using MDRP = typename conv::MDRP; using interface_t = scream::rrtmgp::rrtmgp_interface; +using real1dk = typename interface_t::view_t; +using real2dk = typename interface_t::view_t; +using real3dk = typename interface_t::view_t; +using MDRP = typename conv::MDRP; static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 6e924103229f..8c236e64e811 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -57,9 +57,10 @@ void compute_heating_rate ( View4 const &heating_rate) { using physconst = scream::physics::Constants; + using MDRP = typename conv::MDRP; auto ncol = flux_up.extent(0); auto nlay = flux_up.extent(1)-1; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { heating_rate(icol,ilay) = ( flux_up(icol,ilay+1) - flux_up(icol,ilay) - flux_dn(icol,ilay+1) + flux_dn(icol,ilay) @@ -98,18 +99,20 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } }); auto num_bad = sum(bad_mask); - pass = false; - out << msg << ": " - << num_bad << " values outside range " - << "[" << xmin << "," << xmax << "]" - << "; minval = " << _xmin - << "; maxval = " << _xmax << "\n"; + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } } return pass; } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template ::type* = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; @@ -117,10 +120,10 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - Kokkos::View bad_mask("bad_mask", x.size()); - Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { - if (x.data()[i] < xmin or x.data()[i] > xmax) { - bad_mask.data()[i] = true; + Kokkos::View bad_mask("bad_mask", x.extent(0)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + if (x(i) < xmin or x(i) > xmax) { + bad_mask(i) = true; } }); auto num_bad = conv::sum(bad_mask); @@ -133,6 +136,68 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val } return pass; } + +template ::type* = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + if (x(i, j) < xmin or x(i, j) > xmax) { + bad_mask(i, j) = true; + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + +template ::type* = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1), x.extent(2)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + for (size_t k = 0; k < x.extent(2); ++k) { + if (x(i, j, k) < xmin or x(i, j, k) > xmax) { + bad_mask(i, j, k) = true; + } + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + + #endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 2038cb62ff79..c82d9e38a3b6 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -159,7 +159,7 @@ int get_wavelength_index_lw(double wavelength); // New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS -template +template struct rrtmgp_interface { using MDRP = typename conv::MDRP; @@ -980,7 +980,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i // Kokkos::deep_copy(seeds_host, seeds); // for (int icol = 0; icol < ncol; ++icol) { // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // Kokkos::parallel_for(MDRP::template get<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { // auto generator = random_pool.get_state(); // cldx(icol,ilay,igpt) = generator.drand(0., 1.); // random_pool.free_state(generator); @@ -996,7 +996,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i }); // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { for (int ilay = 1; ilay < nlay; ilay++) { // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { @@ -1016,7 +1016,7 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i } // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { subcolumn_mask(icol,ilay,igpt) = 1; } else { @@ -1202,13 +1202,21 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template +template::type* = nullptr> static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + Kokkos::parallel_for(arr_out.size(), KOKKOS_LAMBDA(int i) { + arr_out(i) = std::min(std::max(arr_in(i), lower), upper); + }); +} + +template::type* = nullptr> +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { + Kokkos::parallel_for(MDRP::template get<2>({arr_out.extent(0), arr_out.extent(1)}), KOKKOS_LAMBDA(int i, int j) { + arr_out(i, j) = std::min(std::max(arr_in(i, j), lower), upper); }); } + static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) { // Get wavelength bounds for all wavelength bands @@ -1382,7 +1390,7 @@ static OpticalProps1sclK get_subsampled_clouds( auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { auto ibnd = gpoint_bands(igpt); if (cldmask(icol,ilay,igpt) == 1) { subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 65d1da9f1d68..0d3f18e7d841 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -320,6 +320,12 @@ int run_yakl(int argc, char** argv) { int run_kokkos(int argc, char** argv) { using namespace ekat::logger; using logger_t = Logger; + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using MDRP = utils_t::MDRP; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; ekat::Comm comm(MPI_COMM_WORLD); auto logger = std::make_shared("",LogLevel::info,comm); @@ -374,7 +380,7 @@ int run_kokkos(int argc, char** argv) { real2dk sw_flux_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Get dimension sizes int ncol = sw_flux_up_ref.extent(0); @@ -394,12 +400,12 @@ int run_kokkos(int argc, char** argv) { real2dk p_lev("p_lev", ncol, nlay+1); real2dk t_lev("t_lev", ncol, nlay+1); real2dk col_dry; - GasConcsK gas_concs; + GasConcsK gas_concs; read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry, ncol); // Initialize absorption coefficients logger->info("Initialize RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Setup our dummy atmosphere based on the input data we read in logger->info("Setup dummy atmos...\n"); @@ -413,7 +419,7 @@ int run_kokkos(int argc, char** argv) { real2dk rel("rel", ncol, nlay); real2dk rei("rei", ncol, nlay); real2dk cld("cld", ncol, nlay); - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol, p_lay, t_lay, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -426,8 +432,8 @@ int run_kokkos(int argc, char** argv) { // we would just have to setup the pointers to them in the // FluxesBroadband object logger->info("Setup fluxes...\n"); - const auto nswbands = scream::rrtmgp::k_dist_sw_k.get_nband(); - const auto nlwbands = scream::rrtmgp::k_dist_lw_k.get_nband(); + const auto nswbands = interface_t::k_dist_sw_k.get_nband(); + const auto nlwbands = interface_t::k_dist_lw_k.get_nband(); real2dk sw_flux_up ("sw_flux_up" , ncol, nlay+1); real2dk sw_flux_dn ("sw_flux_dn" , ncol, nlay+1); real2dk sw_flux_dir("sw_flux_dir", ncol, nlay+1); @@ -457,7 +463,7 @@ int run_kokkos(int argc, char** argv) { // Compute band-by-band surface_albedos. real2dk sfc_alb_dir("sfc_alb_dir", ncol, nswbands); real2dk sfc_alb_dif("sfc_alb_dif", ncol, nswbands); - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -468,19 +474,19 @@ int run_kokkos(int argc, char** argv) { auto aer_ssa_sw = real3dk("aer_ssa_sw", ncol, nlay, nswbands); auto aer_asm_sw = real3dk("aer_asm_sw", ncol, nlay, nswbands); auto aer_tau_lw = real3dk("aer_tau_lw", ncol, nlay, nlwbands); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_sw(icol,ilay,ibnd) = 0; aer_ssa_sw(icol,ilay,ibnd) = 0; aer_asm_sw(icol,ilay,ibnd) = 0; }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_lw(icol,ilay,ibnd) = 0; }); // These are returned as outputs now from rrtmgp_main // TODO: provide as inputs consistent with how aerosol is treated? - const auto nswgpts = scream::rrtmgp::k_dist_sw_k.get_ngpt(); - const auto nlwgpts = scream::rrtmgp::k_dist_lw_k.get_ngpt(); + const auto nswgpts = interface_t::k_dist_sw_k.get_ngpt(); + const auto nlwgpts = interface_t::k_dist_lw_k.get_ngpt(); auto cld_tau_sw_bnd = real3dk("cld_tau_sw_bnd", ncol, nlay, nswbands); auto cld_tau_lw_bnd = real3dk("cld_tau_lw_bnd", ncol, nlay, nlwbands); auto cld_tau_sw = real3dk("cld_tau_sw", ncol, nlay, nswgpts); @@ -489,7 +495,7 @@ int run_kokkos(int argc, char** argv) { // Run RRTMGP code on dummy atmosphere logger->info("Run RRTMGP...\n"); const Real tsi_scaling = 1; - scream::rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, nlay, p_lay, t_lay, p_lev, t_lev, gas_concs, sfc_alb_dir, sfc_alb_dif, mu0, @@ -513,33 +519,33 @@ int run_kokkos(int argc, char** argv) { // Check values against baseline logger->info("Check values...\n"); - rrtmgpTest::read_fluxes( + utils_t::read_fluxes( baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; // Because the aerosol optical properties are all set to zero, these fluxes must be equal - if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; logger->info("Cleaning up...\n"); // Clean up or else YAKL will throw errors - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); scream::finalize_kls(); return nerr != 0 ? 1 : 0; diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 36e2a59e02ff..99a582445616 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -856,6 +856,7 @@ using real3dk = interface_t::view_t; using int1dk = interface_t::view_t; using int2dk = interface_t::view_t; using int3dk = interface_t::view_t; +using MDRP = interface_t::MDRP; TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos @@ -1082,7 +1083,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Need to initialize RRTMGP with dummy gases logger->info("Init gases...\n"); - GasConcsK gas_concs; + GasConcsK gas_concs; string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); @@ -1100,7 +1101,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { auto sw_bnd_flux_dir = real3dk("sw_bnd_flux_dir", ncol, nlay+1, nbnd); auto sw_bnd_flux_dif = real3dk("sw_bnd_flux_dif", ncol, nlay+1, nbnd); logger->info("Populate band-resolved 3d fluxes for test case with only transition band flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1132,7 +1133,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in NIR bands logger->info("Populate band-resolved 3d fluxes for test case with only NIR flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1; sw_bnd_flux_dif(icol,ilay,ibnd) = 1; @@ -1163,7 +1164,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in VIS bands logger->info("Populate band-resolved 3d fluxes for test case with only VIS/UV flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1194,7 +1195,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in all bands logger->info("Populate band-resolved 3d fluxes for test with non-zero flux in all bands...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1.0; sw_bnd_flux_dif(icol,ilay,ibnd) = 2.0; @@ -1293,13 +1294,13 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { for (int igpt = 0; igpt < ngpt; ++igpt) { real cldmask_real = cldmask(icol,ilay,igpt); cldfrac_from_mask(icol,ilay) += cldmask_real; } }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { cldfrac_from_mask(icol,ilay) = cldfrac_from_mask(icol,ilay) / ngpt; }); // For cldfrac 1 we should get 1, for cldfrac 0 we should get 0, but in between we cannot be sure diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 9e6e94a2164f..75feeb1b8dd4 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -422,6 +422,13 @@ TEST_CASE("rrtmgp_scream_standalone", "") { #endif #ifdef RRTMGP_ENABLE_KOKKOS TEST_CASE("rrtmgp_scream_standalone_k", "") { + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using MDRP = interface_t::MDRP; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; + // Get baseline name (needs to be passed as an arg) std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); @@ -439,7 +446,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { real2dk sw_flux_dn_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Load ad parameter list std::string fname = "input_unit.yaml"; @@ -526,11 +533,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { // this will copy the first column of the input data (the first profile) ncol // times. We will then fill some fraction of these columns with clouds for // the test problem. - GasConcsK gas_concs; + GasConcsK gas_concs; real2dk col_dry; read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, col_dry, ncol_all); // Setup dummy problem; need to use tmp arrays with ncol_all size - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol_all, p_lay_all, t_lay_all, sfc_alb_dir_vis_all, sfc_alb_dir_nir_all, sfc_alb_dif_vis_all, sfc_alb_dif_nir_all, @@ -547,7 +554,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { sfc_alb_dif_nir(icol) = sfc_alb_dif_nir_all(icol_all); mu0(icol) = mu0_all(icol_all); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lay(icol,ilay) = p_lay_all(icol_all,ilay); t_lay(icol,ilay) = t_lay_all(icol_all,ilay); @@ -557,14 +564,14 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { rei(icol,ilay) = rei_all(icol_all,ilay); cld(icol,ilay) = cld_all(icol_all,ilay); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lev(icol,ilay) = p_lev_all(icol_all,ilay); t_lev(icol,ilay) = t_lev_all(icol_all,ilay); }); // Need to calculate a dummy pseudo_density for our test problem - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { p_del(icol,ilay) = abs(p_lev(icol,ilay+1) - p_lev(icol,ilay)); }); @@ -573,13 +580,13 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto qc = real2dk("qc", ncol, nlay); auto nc = real2dk("nc", ncol, nlay); auto qi = real2dk("qi", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); qi(icol,ilay) = 1e-3 * iwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); }); // Copy gases from gas_concs to gas_vmr array - Kokkos::parallel_for(conv::get_mdrp<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { + Kokkos::parallel_for(MDRP::template get<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { auto icol_all = ncol * irank + icol; gas_vmr(icol,ilay,igas) = gas_concs.concs(icol_all,ilay,igas); }); @@ -691,7 +698,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { const int i = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - if (k < nlay) t_lay(i+1,k+1) = d_tmid(i,k); + if (k < nlay) t_lay(i,k) = d_tmid(i,k); sw_flux_up_test(i,k) = d_sw_flux_up(i,k); sw_flux_dn_test(i,k) = d_sw_flux_dn(i,k); @@ -722,7 +729,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto sw_flux_dn_dir_loc = real2dk("sw_flux_dn_dir_loc", ncol, nlay+1); auto lw_flux_up_loc = real2dk("lw_flux_up_loc" , ncol, nlay+1); auto lw_flux_dn_loc = real2dk("lw_flux_dn_loc" , ncol, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; sw_flux_up_loc(icol,ilay) = sw_flux_up_ref(icol_all,ilay); sw_flux_dn_loc(icol,ilay) = sw_flux_dn_ref(icol_all,ilay); @@ -730,11 +737,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { lw_flux_up_loc(icol,ilay) = lw_flux_up_ref(icol_all,ilay); lw_flux_dn_loc(icol,ilay) = lw_flux_dn_ref(icol_all,ilay); }); - REQUIRE(rrtmgpTest::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); + REQUIRE(utils_t::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); // Finalize the driver. YAKL will be finalized inside // RRTMGPRadiation::finalize_impl after RRTMGP has had the From b7e006d8070926c19c218b940398adf225052578 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 14:50:19 -0600 Subject: [PATCH 250/477] cleanup --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 89 ++++++------------- 1 file changed, 27 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1556280eab4b..cde298cc8cd9 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -19,28 +19,6 @@ namespace scream { -template -void check_views2(const View1& view1, const View2& view2) -{ - constexpr auto krank1 = View1::rank; - constexpr auto krank2 = View2::rank; - - RRT_REQUIRE(krank1 == krank2, "Rank mismatch for: " << view1.label()); - RRT_REQUIRE(krank1 == 2, "Rank mismatch for: " << view1.label()); - - for (auto r = 0; r < krank1; ++r) { - RRT_REQUIRE(view1.extent(r) == view2.extent(r), "Dim mismatch for: " << view1.label() << ", rank: " << r << ", " << view1.extent(r) << " != " << view2.extent(r)); - } - - for (auto i = 0; i < view1.extent(0); ++i) { - for (auto j = 0; j < view1.extent(1); ++j) { - const auto data1 = view1(i, j); - const auto data2 = view2(i, j); - RRT_REQUIRE(data1 == data2, "Data mismatch for: " << view1.label() << ", i: " << i << ", j: " << j << ", " << data1 << " != " << data2); - } - } -} - using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; @@ -967,8 +945,8 @@ void RRTMGPRadiation::run_impl (const double dt) { return subv; #endif }; - auto subview_2dk = [&](const ureal2dk& v) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); + auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { + ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); #ifdef RRTMGP_ENABLE_YAKL real2dk rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); @@ -985,16 +963,6 @@ void RRTMGPRadiation::run_impl (const double dt) { return rv; #else return subv; -#endif - }; - auto subview_2dk_new = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); -#ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; #endif }; auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { @@ -1027,26 +995,26 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; auto rel_k = subview_2dkc(d_rel, m_nlay); auto rei_k = subview_2dkc(d_rei, m_nlay); - auto sw_flux_up_k = subview_2dk_new(d_sw_flux_up, m_nlay+1); - auto sw_flux_dn_k = subview_2dk_new(d_sw_flux_dn, m_nlay+1); - auto sw_flux_dn_dir_k = subview_2dk_new(d_sw_flux_dn_dir, m_nlay+1); - auto lw_flux_up_k = subview_2dk_new(d_lw_flux_up, m_nlay+1); - auto lw_flux_dn_k = subview_2dk_new(d_lw_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_up_k = subview_2dk_new(d_sw_clnclrsky_flux_up, m_nlay+1); - auto sw_clnclrsky_flux_dn_k = subview_2dk_new(d_sw_clnclrsky_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); - auto sw_clrsky_flux_up_k = subview_2dk_new(d_sw_clrsky_flux_up, m_nlay+1); - auto sw_clrsky_flux_dn_k = subview_2dk_new(d_sw_clrsky_flux_dn, m_nlay+1); - auto sw_clrsky_flux_dn_dir_k = subview_2dk_new(d_sw_clrsky_flux_dn_dir, m_nlay+1); - auto sw_clnsky_flux_up_k = subview_2dk_new(d_sw_clnsky_flux_up, m_nlay+1); - auto sw_clnsky_flux_dn_k = subview_2dk_new(d_sw_clnsky_flux_dn, m_nlay+1); - auto sw_clnsky_flux_dn_dir_k = subview_2dk_new(d_sw_clnsky_flux_dn_dir, m_nlay+1); - auto lw_clnclrsky_flux_up_k = subview_2dk_new(d_lw_clnclrsky_flux_up, m_nlay+1); - auto lw_clnclrsky_flux_dn_k = subview_2dk_new(d_lw_clnclrsky_flux_dn, m_nlay+1); - auto lw_clrsky_flux_up_k = subview_2dk_new(d_lw_clrsky_flux_up, m_nlay+1); - auto lw_clrsky_flux_dn_k = subview_2dk_new(d_lw_clrsky_flux_dn, m_nlay+1); - auto lw_clnsky_flux_up_k = subview_2dk_new(d_lw_clnsky_flux_up, m_nlay+1); - auto lw_clnsky_flux_dn_k = subview_2dk_new(d_lw_clnsky_flux_dn, m_nlay+1); + auto sw_flux_up_k = subview_2dk(d_sw_flux_up, m_nlay+1); + auto sw_flux_dn_k = subview_2dk(d_sw_flux_dn, m_nlay+1); + auto sw_flux_dn_dir_k = subview_2dk(d_sw_flux_dn_dir, m_nlay+1); + auto lw_flux_up_k = subview_2dk(d_lw_flux_up, m_nlay+1); + auto lw_flux_dn_k = subview_2dk(d_lw_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_up_k = subview_2dk(d_sw_clnclrsky_flux_up, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = subview_2dk(d_sw_clnclrsky_flux_dn, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); + auto sw_clrsky_flux_up_k = subview_2dk(d_sw_clrsky_flux_up, m_nlay+1); + auto sw_clrsky_flux_dn_k = subview_2dk(d_sw_clrsky_flux_dn, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = subview_2dk(d_sw_clrsky_flux_dn_dir, m_nlay+1); + auto sw_clnsky_flux_up_k = subview_2dk(d_sw_clnsky_flux_up, m_nlay+1); + auto sw_clnsky_flux_dn_k = subview_2dk(d_sw_clnsky_flux_dn, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = subview_2dk(d_sw_clnsky_flux_dn_dir, m_nlay+1); + auto lw_clnclrsky_flux_up_k = subview_2dk(d_lw_clnclrsky_flux_up, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = subview_2dk(d_lw_clnclrsky_flux_dn, m_nlay+1); + auto lw_clrsky_flux_up_k = subview_2dk(d_lw_clrsky_flux_up, m_nlay+1); + auto lw_clrsky_flux_dn_k = subview_2dk(d_lw_clrsky_flux_dn, m_nlay+1); + auto lw_clnsky_flux_up_k = subview_2dk(d_lw_clnsky_flux_up, m_nlay+1); + auto lw_clnsky_flux_dn_k = subview_2dk(d_lw_clnsky_flux_dn, m_nlay+1); auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; @@ -1224,9 +1192,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // correct size that uses m_buffer.tmp2d's pointer #ifdef RRTMGP_ENABLE_YAKL real2d tmp2d = subview_2d(m_buffer.tmp2d); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); #endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; @@ -1235,22 +1200,22 @@ void RRTMGPRadiation::run_impl (const double dt) { // 'o3' is marked as 'Required' rather than 'Computed', so we need to get the proper field auto f = name=="o3" ? get_field_in(full_name) : get_field_out(full_name); auto d_vmr = f.get_view(); +#ifdef RRTMGP_ENABLE_KOKKOS + auto tmp2d_k = subview_2dkc(d_vmr, m_nlay); +#endif +#ifdef RRTMGP_ENABLE_YAKL // Copy to YAKL const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { -#ifdef RRTMGP_ENABLE_YAKL tmp2d(i+1,k+1) = d_vmr(icol,k); // Note that for YAKL arrays i and k start with index 1 -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - tmp2d_k(i,k) = d_vmr(icol,k); -#endif }); }); Kokkos::fence(); +#endif // Populate GasConcs object #ifdef RRTMGP_ENABLE_YAKL From f1e2d98e68a43e98cf212e1ae5c4d70254963a75 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 15:48:32 -0600 Subject: [PATCH 251/477] subview_2dk bug fix --- .../eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index cde298cc8cd9..9fab85444fcc 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -946,7 +946,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(0, ncol), std::make_pair(0, inner_dim)); + ureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); #ifdef RRTMGP_ENABLE_YAKL real2dk rv(v.label(), ncol, v.extent(1)); Kokkos::deep_copy(rv, subv); From 23f89ce9d5a34e15d01857492a369e550af77185 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 20 Jun 2024 17:44:50 -0600 Subject: [PATCH 252/477] More fixes --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 9fab85444fcc..7a5af5843d19 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -926,7 +926,7 @@ void RRTMGPRadiation::run_impl (const double dt) { // check_range_k will fail due to looking at unused values. Once rrtmgp can handle // packs, this won't be necessary. auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { - ureal1dk subv(v, std::make_pair(0, ncol)); + ureal1dk subv(v, std::make_pair(beg, beg+ncol)); #ifdef RRTMGP_ENABLE_YAKL real1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); @@ -936,7 +936,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { - cureal1dk subv(v, std::make_pair(0, ncol)); + cureal1dk subv(v, std::make_pair(beg, beg+ncol)); #ifdef RRTMGP_ENABLE_YAKL creal1dk rv(v.label(), ncol); Kokkos::deep_copy(rv, subv); @@ -966,7 +966,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif }; auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { - ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); // The range assumes these are buffer views, not fields #ifdef RRTMGP_ENABLE_YAKL real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); Kokkos::deep_copy(rv, subv); From 8cae9b12fc801bb9649c6182ac85fd4194ccb546 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 13:04:44 -0600 Subject: [PATCH 253/477] Set to correct rrtmgp branch --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 4c542032b3ac..02c2e03df33e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 4c542032b3ac3f842a9619c9eadc005c507a0366 +Subproject commit 02c2e03df33edca6101670168ca187cb94c0768e From 0eddf307ad51587d4bc1a98bed8077e614e0c174 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 14:01:18 -0600 Subject: [PATCH 254/477] Add buffer views should be unmanaged --- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index c60a1cb3181a..5dece435fb0a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -145,15 +145,15 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + ureal1dk mu0_k; + ureal1dk sfc_alb_dir_vis_k; + ureal1dk sfc_alb_dir_nir_k; + ureal1dk sfc_alb_dif_vis_k; + ureal1dk sfc_alb_dif_nir_k; + ureal1dk sfc_flux_dir_vis_k; + ureal1dk sfc_flux_dir_nir_k; + ureal1dk sfc_flux_dif_vis_k; + ureal1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) @@ -176,21 +176,21 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + ureal2dk p_lay_k; + ureal2dk t_lay_k; + ureal2dk z_del_k; + ureal2dk p_del_k; + ureal2dk qc_k; + ureal2dk nc_k; + ureal2dk qi_k; + ureal2dk cldfrac_tot_k; + ureal2dk eff_radius_qc_k; + ureal2dk eff_radius_qi_k; + ureal2dk tmp2d_k; + ureal2dk lwp_k; + ureal2dk iwp_k; + ureal2dk sw_heating_k; + ureal2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) @@ -220,28 +220,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + ureal2dk p_lev_k; + ureal2dk t_lev_k; + ureal2dk sw_flux_up_k; + ureal2dk sw_flux_dn_k; + ureal2dk sw_flux_dn_dir_k; + ureal2dk lw_flux_up_k; + ureal2dk lw_flux_dn_k; + ureal2dk sw_clnclrsky_flux_up_k; + ureal2dk sw_clnclrsky_flux_dn_k; + ureal2dk sw_clnclrsky_flux_dn_dir_k; + ureal2dk sw_clrsky_flux_up_k; + ureal2dk sw_clrsky_flux_dn_k; + ureal2dk sw_clrsky_flux_dn_dir_k; + ureal2dk sw_clnsky_flux_up_k; + ureal2dk sw_clnsky_flux_dn_k; + ureal2dk sw_clnsky_flux_dn_dir_k; + ureal2dk lw_clnclrsky_flux_up_k; + ureal2dk lw_clnclrsky_flux_dn_k; + ureal2dk lw_clrsky_flux_up_k; + ureal2dk lw_clrsky_flux_dn_k; + ureal2dk lw_clnsky_flux_up_k; + ureal2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -252,10 +252,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + ureal3dk sw_bnd_flux_up_k; + ureal3dk sw_bnd_flux_dn_k; + ureal3dk sw_bnd_flux_dir_k; + ureal3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -264,8 +264,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + ureal3dk lw_bnd_flux_up_k; + ureal3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -274,8 +274,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + ureal2dk sfc_alb_dir_k; + ureal2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -286,10 +286,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + ureal3dk aero_tau_sw_k; + ureal3dk aero_ssa_sw_k; + ureal3dk aero_g_sw_k; + ureal3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -298,8 +298,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + ureal3dk cld_tau_sw_bnd_k; + ureal3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -308,8 +308,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + ureal3dk cld_tau_sw_gpt_k; + ureal3dk cld_tau_lw_gpt_k; #endif }; From 5efd49e03c3f097723de4db33f72483dd14769ec Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 21 Jun 2024 15:55:26 -0600 Subject: [PATCH 255/477] Build fixes for yakl mode --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 4 ++-- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 7a5af5843d19..62de7f8416cd 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -1293,8 +1293,8 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - interface_t::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - interface_t::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 5dece435fb0a..e036e7ac08c0 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -34,7 +34,9 @@ class RRTMGPRadiation : public AtmosphereProcess { using layout_t = typename ekat::KokkosTypes::Layout; +#ifdef RRTMGP_ENABLE_KOKKOS using interface_t = rrtmgp::rrtmgp_interface; +#endif // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); From dd83e12a9e977da10104e0b4338cffb3e8241885 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 26 Jul 2024 13:04:18 -0600 Subject: [PATCH 256/477] Remove overuse of templates --- .../rrtmgp/scream_rrtmgp_interface.hpp | 234 ++++++++---------- 1 file changed, 109 insertions(+), 125 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index c82d9e38a3b6..0bceb1282e5f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -175,21 +175,40 @@ using hview_t = Kokkos::View; using pool_t = conv::MemPoolSingleton; +using real1dk = view_t; +using real2dk = view_t; +using real3dk = view_t; +using creal1dk = view_t; +using creal2dk = view_t; +using creal3dk = view_t; +using int1dk = view_t; +using int3dk = view_t; + +using gas_optics_t = GasOpticsRRTMGPK; +using cloud_optics_t = CloudOpticsK; +using gas_concs_t = GasConcsK; +using fluxes_t = FluxesBybandK; +using fluxes_broadband_t = FluxesBroadbandK; +using optical_props_t = OpticalPropsK; +using optical_props1_t = OpticalProps1sclK; +using optical_props2_t = OpticalProps2strK; +using source_func_t = SourceFuncLWK; + /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -static inline GasOpticsRRTMGPK k_dist_sw_k; -static inline GasOpticsRRTMGPK k_dist_lw_k; +static inline gas_optics_t k_dist_sw_k; +static inline gas_optics_t k_dist_lw_k; /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -static inline CloudOpticsK cloud_optics_sw_k; -static inline CloudOpticsK cloud_optics_lw_k; +static inline cloud_optics_t cloud_optics_sw_k; +static inline cloud_optics_t cloud_optics_lw_k; /* * Flag to indicate whether or not we have initialized RRTMGP @@ -200,7 +219,7 @@ static inline bool initialized_k = false; * Initialize data for RRTMGP driver */ static void rrtmgp_initialize( - GasConcsK &gas_concs, + const gas_concs_t &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger) @@ -238,13 +257,11 @@ static void rrtmgp_initialize( /* * Compute band-by-band surface albedos from broadband albedos. */ -template static void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, - SfcAlbDirVisT &sfc_alb_dir_vis, SfcAlbDirNirT &sfc_alb_dir_nir, - SfcAlbDifVisT &sfc_alb_dif_vis, SfcAlbDifNirT &sfc_alb_dif_nir, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif) + const creal1dk &sfc_alb_dir_vis, const creal1dk &sfc_alb_dir_nir, + const creal1dk &sfc_alb_dif_vis, const creal1dk &sfc_alb_dif_nir, + const real2dk &sfc_alb_dir, const real2dk &sfc_alb_dif) { EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); @@ -289,13 +306,11 @@ static void compute_band_by_band_surface_albedos( /* * Compute broadband visible/UV and near-infrared surface fluxes. */ -template static void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, - SwBndFluxDirT &sw_bnd_flux_dir , SwBndFluxDifT &sw_bnd_flux_dif , - SfcFluxDirVisT &sfc_flux_dir_vis, SfcFluxDirNirT &sfc_flux_dir_nir, - SfcFluxDifVisT &sfc_flux_dif_vis, SfcFluxDifNirT &sfc_flux_dif_nir) + const real3dk &sw_bnd_flux_dir , const real3dk &sw_bnd_flux_dif , + const real1dk &sfc_flux_dir_vis, const real1dk &sfc_flux_dir_nir, + const real1dk &sfc_flux_dif_vis, const real1dk &sfc_flux_dif_nir) { // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when @@ -349,42 +364,26 @@ static void compute_broadband_surface_fluxes( * The input logger is in charge of outputing info to * screen and/or to file (or neither), depending on how it was set up. */ -template static void rrtmgp_main( const int ncol, const int nlay, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei, CldfracT &cldfrac, - AerTauSwT &aer_tau_sw, AerSsaSwT &aer_ssa_sw, AerAsmSwT &aer_asm_sw, AerTauLwT &aer_tau_lw, - CldTauSwBndT &cld_tau_sw_bnd, CldTauLwBndT &cld_tau_lw_bnd, - CldTauSwGptT &cld_tau_sw_gpt, CldTauLwGptT &cld_tau_lw_gpt, - SwFluxUpT &sw_flux_up, SwFluxDnT &sw_flux_dn, SwFluxDnDirT &sw_flux_dn_dir, - LwFluxUpT &lw_flux_up, LwFluxDnT &lw_flux_dn, - SwClnclrskyFluxUpT &sw_clnclrsky_flux_up, SwClnclrskyFluxDnT &sw_clnclrsky_flux_dn, SwClnclrskyFluxDnDirT &sw_clnclrsky_flux_dn_dir, - SwClrskyFluxUpT &sw_clrsky_flux_up, SwClrskyFluxDnT &sw_clrsky_flux_dn, SwClrskyFluxDnDirT &sw_clrsky_flux_dn_dir, - SwClnskyFluxUpT &sw_clnsky_flux_up, SwClnskyFluxDnT &sw_clnsky_flux_dn, SwClnskyFluxDnDirT &sw_clnsky_flux_dn_dir, - LwClnclrskyFluxUpT &lw_clnclrsky_flux_up, LwClnclrskyFluxDnT &lw_clnclrsky_flux_dn, - LwClrskyFluxUpT &lw_clrsky_flux_up, LwClrskyFluxDnT &lw_clrsky_flux_dn, - LwClnskyFluxUpT &lw_clnsky_flux_up, LwClnskyFluxDnT &lw_clnsky_flux_dn, - SwBndFluxUpT &sw_bnd_flux_up, SwBndFluxDnT &sw_bnd_flux_dn, SwBndFluxDnDirT &sw_bnd_flux_dn_dir, - LwBndFluxUpT &lw_bnd_flux_up, LwBndFluxDnT &lw_bnd_flux_dn, - const RealT tsi_scaling, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei, const real2dk &cldfrac, + const real3dk &aer_tau_sw, const real3dk &aer_ssa_sw, const real3dk &aer_asm_sw, const real3dk &aer_tau_lw, + const real3dk &cld_tau_sw_bnd, const real3dk &cld_tau_lw_bnd, + const real3dk &cld_tau_sw_gpt, const real3dk &cld_tau_lw_gpt, + const real2dk &sw_flux_up, const real2dk &sw_flux_dn, const real2dk &sw_flux_dn_dir, + const real2dk &lw_flux_up, const real2dk &lw_flux_dn, + const real2dk &sw_clnclrsky_flux_up, const real2dk &sw_clnclrsky_flux_dn, const real2dk &sw_clnclrsky_flux_dn_dir, + const real2dk &sw_clrsky_flux_up, const real2dk &sw_clrsky_flux_dn, const real2dk &sw_clrsky_flux_dn_dir, + const real2dk &sw_clnsky_flux_up, const real2dk &sw_clnsky_flux_dn, const real2dk &sw_clnsky_flux_dn_dir, + const real2dk &lw_clnclrsky_flux_up, const real2dk &lw_clnclrsky_flux_dn, + const real2dk &lw_clrsky_flux_up, const real2dk &lw_clrsky_flux_dn, + const real2dk &lw_clnsky_flux_up, const real2dk &lw_clnsky_flux_dn, + const real3dk &sw_bnd_flux_up, const real3dk &sw_bnd_flux_dn, const real3dk &sw_bnd_flux_dn_dir, + const real3dk &lw_bnd_flux_up, const real3dk &lw_bnd_flux_dn, + const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) { @@ -404,7 +403,7 @@ static void rrtmgp_main( #endif // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; + fluxes_t fluxes_sw; fluxes_sw.flux_up = sw_flux_up; fluxes_sw.flux_dn = sw_flux_dn; fluxes_sw.flux_dn_dir = sw_flux_dn_dir; @@ -412,37 +411,37 @@ static void rrtmgp_main( fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; + fluxes_broadband_t clnclrsky_fluxes_sw; clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; + fluxes_broadband_t clrsky_fluxes_sw; clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; + fluxes_broadband_t clnsky_fluxes_sw; clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; + fluxes_t fluxes_lw; fluxes_lw.flux_up = lw_flux_up; fluxes_lw.flux_dn = lw_flux_dn; fluxes_lw.bnd_flux_up = lw_bnd_flux_up; fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; + fluxes_broadband_t clnclrsky_fluxes_lw; clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; + fluxes_broadband_t clrsky_fluxes_lw; clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; + fluxes_broadband_t clnsky_fluxes_lw; clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; @@ -450,8 +449,8 @@ static void rrtmgp_main( auto nlwbands = k_dist_lw_k.get_nband(); // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; + optical_props2_t aerosol_sw; + optical_props1_t aerosol_lw; aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); aerosol_sw.alloc_2str(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { @@ -476,8 +475,8 @@ static void rrtmgp_main( #endif // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + optical_props2_t clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + optical_props1_t clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); @@ -549,17 +548,15 @@ static void rrtmgp_finalize() /* * Shortwave driver (called by rrtmgp_main) */ -template static void rrtmgp_sw( const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - SfcAlbDirT &sfc_alb_dir, SfcAlbDifT &sfc_alb_dif, Mu0T &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const RealT tsi_scaling, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + optical_props2_t &aerosol, optical_props2_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, + const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { @@ -647,7 +644,7 @@ static void rrtmgp_sw( // Subset gases auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; + gas_concs_t gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { auto vmr_day = view_t("vmr_day", nday, nlay); @@ -660,7 +657,7 @@ static void rrtmgp_sw( } // Subset aerosol optics - OpticalProps2strK aerosol_day; + optical_props2_t aerosol_day; aerosol_day.init(k_dist.get_band_lims_wavenumber()); aerosol_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { @@ -671,7 +668,7 @@ static void rrtmgp_sw( // Subset cloud optics // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; + optical_props2_t clouds_day; clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); clouds_day.alloc_2str(nday, nlay); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { @@ -697,7 +694,7 @@ static void rrtmgp_sw( auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; + fluxes_t fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; fluxes_day.flux_dn_dir = flux_dn_dir_day; @@ -706,10 +703,10 @@ static void rrtmgp_sw( fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; // Allocate space for optical properties - OpticalProps2strK optics; + optical_props2_t optics; optics.alloc_2str(nday, nlay, k_dist); - OpticalProps2strK optics_no_aerosols; + optical_props2_t optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_2str(nday, nlay, k_dist); @@ -811,14 +808,13 @@ static void rrtmgp_sw( /* * Longwave driver (called by rrtmgp_main) */ -template static void rrtmgp_lw( const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - PlayT &p_lay, TlayT &t_lay, PlevT &p_lev, TlevT &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + optical_props1_t &aerosol, optical_props1_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Problem size @@ -856,16 +852,16 @@ static void rrtmgp_lw( }); // Allocate space for optical properties - OpticalProps1sclK optics; + optical_props1_t optics; optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; + optical_props1_t optics_no_aerosols; if (extra_clnsky_diag) { // Allocate space for optical properties (no aerosols) optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); } // Boundary conditions - SourceFuncLWK lw_sources; + source_func_t lw_sources; lw_sources.alloc(ncol, nlay, k_dist); view_t t_sfc ("t_sfc" ,ncol); view_t emis_sfc("emis_sfc",nbnd,ncol); @@ -950,12 +946,10 @@ static void rrtmgp_lw( /* * Return a subcolumn mask consistent with a specified overlap assumption */ -template -static view_t get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, - CldfT &cldf, const int overlap_option, SeedsT &seeds) +static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds) { // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = view_t("subcolumn_mask", ncol, nlay, ngpt); + int3dk subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); // Subcolumn generators are a means for producing a variable x(i,j,k), where // @@ -1029,10 +1023,9 @@ static view_t get_subcolumn_mask(const int ncol, const int nlay, const i /* * Compute cloud area from 3d subcol cloud property */ -template static void compute_cloud_area( - int ncol, int nlay, int ngpt, RealT pmin, RealT pmax, - const PmidT& pmid, const CldTauGptT& cld_tau_gpt, CldAreaT& cld_area) + int ncol, int nlay, int ngpt, Real pmin, Real pmax, + const creal2dk& pmid, const real3dk& cld_tau_gpt, const real1dk& cld_area) { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 @@ -1058,20 +1051,15 @@ static void compute_cloud_area( /* * Return select cloud-top diagnostics following AeroCom recommendation */ -template static void compute_aerocom_cloudtop( - int ncol, int nlay, const TmidT &tmid, const PmidT &pmid, - const PdelT &p_del, const ZdelT &z_del, const QcT &qc, - const QiT &qi, const RelT &rel, const ReiT &rei, - const CldfracTotT &cldfrac_tot, const NcT &nc, - TmidAtCldtopT &T_mid_at_cldtop, PmidAtCldtopT &p_mid_at_cldtop, - CldfracIceAtCldtopT &cldfrac_ice_at_cldtop, CldfracLiqAtCldtopT &cldfrac_liq_at_cldtop, - CldfracTotAtCldtopT &cldfrac_tot_at_cldtop, CdncAtCldtopT &cdnc_at_cldtop, - EffRadiusQcAtCldtopT &eff_radius_qc_at_cldtop, EffRadiusQiAtCldTopT &eff_radius_qi_at_cldtop) + int ncol, int nlay, const creal2dk &tmid, const creal2dk &pmid, + const creal2dk &p_del, const real2dk &z_del, const creal2dk &qc, + const creal2dk &qi, const creal2dk &rel, const creal2dk &rei, + const real2dk &cldfrac_tot, const creal2dk &nc, + const real1dk &T_mid_at_cldtop, const real1dk &p_mid_at_cldtop, + const real1dk &cldfrac_ice_at_cldtop, const real1dk &cldfrac_liq_at_cldtop, + const real1dk &cldfrac_tot_at_cldtop, const real1dk &cdnc_at_cldtop, + const real1dk &eff_radius_qc_at_cldtop, const real1dk &eff_radius_qi_at_cldtop) { /* The goal of this routine is to calculate properties at cloud top * based on the AeroCom recommendation. See reference for routine @@ -1217,7 +1205,7 @@ static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, O } -static int get_wavelength_index(OpticalPropsK &kdist, RealT wavelength) +static int get_wavelength_index(optical_props_t &kdist, RealT wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -1251,14 +1239,13 @@ static inline int get_wavelength_index_lw_k(RealT wavelength) return get_wavelength_index(k_dist_lw_k, wavelength); } -template -static OpticalProps2strK get_cloud_optics_sw( +static optical_props2_t get_cloud_optics_sw( const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { // Initialize optics - OpticalProps2strK clouds; + optical_props2_t clouds; clouds.init(kdist.get_band_lims_wavenumber()); clouds.alloc_2str(ncol, nlay); @@ -1278,14 +1265,13 @@ static OpticalProps2strK get_cloud_optics_sw( return clouds; } -template -static OpticalProps1sclK get_cloud_optics_lw( +static optical_props1_t get_cloud_optics_lw( const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - LwpT &lwp, IwpT &iwp, RelT &rel, ReiT &rei) { + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { // Initialize optics - OpticalProps1sclK clouds; + optical_props1_t clouds; clouds.init(kdist.get_band_lims_wavenumber()); clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! @@ -1305,12 +1291,11 @@ static OpticalProps1sclK get_cloud_optics_lw( return clouds; } -template -static OpticalProps2strK get_subsampled_clouds( +static optical_props2_t get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, CldT &cld, PlayT &p_lay) { + optical_props2_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { // Initialized subsampled optics - OpticalProps2strK subsampled_optics; + optical_props2_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_2str(ncol, nlay); // Check that we do not have clouds with no optical properties; this would get corrected @@ -1356,13 +1341,12 @@ static OpticalProps2strK get_subsampled_clouds( return subsampled_optics; } -template -static OpticalProps1sclK get_subsampled_clouds( +static optical_props1_t get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, - CldT &cld, PlayT &p_lay) { + optical_props1_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { + // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; + optical_props1_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_1scl(ncol, nlay); // Check that we do not have clouds with no optical properties; this would get corrected From eb6bcca697ccadb0fee647091d9f44520d970b74 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 30 Jul 2024 17:19:29 -0600 Subject: [PATCH 257/477] EAMxx: add routine to reset unlimited dim length This allows future snapshots to effectively overwrite what was already stored --- .../src/share/io/scream_scorpio_interface.cpp | 24 +++++++++++++++++++ .../src/share/io/scream_scorpio_interface.hpp | 1 + 2 files changed, 25 insertions(+) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index cb2b501f2b5c..8d2f64994ddd 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -717,6 +717,30 @@ std::string get_time_name (const std::string& filename) return pf.file->time_dim->name; } +void reset_unlimited_dim_len(const std::string& filename, const int new_length) +{ + auto& f = impl::get_file(filename,"scorpio::reset_unlimited_dim_len"); + + // Reset dim length + EKAT_REQUIRE_MSG (f.time_dim!=nullptr, + "Error! Cannot reset unlimited dim length. No unlimited dim stored.\n" + " - file name: " + filename + "\n"); + EKAT_REQUIRE_MSG (new_lengthlength, + "Error! New time dimension length must be shorter than the current one.\n" + " - file name: " + filename + "\n" + " - curr len : " + std::to_string(f.time_dim->length) + "\n" + " - new len : " + std::to_string(new_length) + "\n"); + f.time_dim->length = new_length; + + // Reset number of records counter for each time dep var + for (auto it : f.vars) { + auto& v = *it.second; + if (v.time_dep) { + v.num_records = new_length; + } + } +} + // =================== Decompositions operations ==================== // // NOTES: diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 549de680fe45..54e3d9027651 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -108,6 +108,7 @@ bool is_dim_unlimited (const std::string& filename, // NOTE: these throw if time dim is not present. Use has_dim to check first. int get_time_len (const std::string& filename); std::string get_time_name (const std::string& filename); +void reset_unlimited_dim_len(const std::string& filename, const int new_length); // =================== Decompositions operations ==================== // From 0bdead86c0f3a3b240d92efd3076a62cdc67dd65 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 30 Jul 2024 17:21:19 -0600 Subject: [PATCH 258/477] EAMxx: two fixes to hist restart logic * Only open last output file if the next write timestamp will fit * When resuming a file, reset time dim length to what was stored in the rhist file (will overwrite any timestamp that was written to file after rhist was written) --- .../src/share/io/scream_output_manager.cpp | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 13e2e97101b1..d6be54d09181 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -242,17 +242,18 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const auto& last_output_filename = get_attribute(rhist_file,"GLOBAL","last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { - scorpio::register_file(last_output_filename,scorpio::Read,m_output_file_specs.iotype); - int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); - scorpio::release_file(last_output_filename); + int num_snaps = scorpio::get_attribute(rhist_file,"GLOBAL","last_output_file_num_snaps"); m_output_file_specs.filename = last_output_filename; m_output_file_specs.is_open = true; m_output_file_specs.storage.num_snapshots_in_file = num_snaps; - // The setup_file call will not register any new variable (the file is in Append mode, - // so all dims/vars must already be in the file). However, it will register decompositions, - // since those are a property of the run, not of the file. - setup_file(m_output_file_specs,m_output_control); + + if (m_output_file_specs.storage.snapshot_fits(m_output_control.next_write_ts)) { + // The setup_file call will not register any new variable (the file is in Append mode, + // so all dims/vars must already be in the file). However, it will register decompositions, + // since those are a property of the run, not of the file. + setup_file(m_output_file_specs,m_output_control); + } } scorpio::release_file(rhist_file); } @@ -494,6 +495,9 @@ void OutputManager::run(const util::TimeStamp& timestamp) write_timestamp (filespecs.filename,"last_write",m_output_control.last_write_ts,true); scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_filename",m_output_file_specs.filename); scorpio::set_attribute (filespecs.filename,"GLOBAL","num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); + + int nsnaps = scorpio::get_dimlen(m_output_file_specs.filename,"time"); + scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_file_num_snaps",nsnaps); } // Write these in both output and rhist file. The former, b/c we need these info when we postprocess // output, and the latter b/c we want to make sure these params don't change across restarts @@ -789,6 +793,24 @@ setup_file ( IOFileSpecs& filespecs, auto mode = m_resume_output_file ? scorpio::Append : scorpio::Write; scorpio::register_file(filename,mode,filespecs.iotype); if (m_resume_output_file) { + // We may have resumed an output file that contains extra snapshots *after* the restart time. + // E.g., if we output every step and the run crashed a few steps after writing the restart. + // In that case, we need to reset the time dimension in the output file, so that the extra + // snapshots will be overwritten. + const auto all_times = scorpio::get_all_times(filename); + int ntimes = all_times.size(); + int ngood = 0; + for (const auto& t : all_times) { + auto keep = t<=m_output_control.last_write_ts.days_from(m_case_t0); + if (keep) { + ++ngood; + } else { + break; + } + } + if (ngood Date: Tue, 30 Jul 2024 17:01:29 -0700 Subject: [PATCH 259/477] fixes to comble test case and update comments for dycomsrf01 --- .../testdefs/testmods_dirs/scream/dpxx/comble/shell_commands | 4 ++-- .../testmods_dirs/scream/dpxx/dycomsrf01/shell_commands | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands index 166e77b1f4bb..5b3b6989aa41 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -5,7 +5,7 @@ ./xmlchange PTS_LAT=74.5 ./xmlchange PTS_LON=10.0 -./xmlchange SSTICE_DATA_FILENAME="${DIN_LOC_ROOT}/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc" +./xmlchange SSTICE_DATA_FILENAME='$DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc' ./xmlchange SSTICE_YEAR_ALIGN=2020 ./xmlchange SSTICE_YEAR_START=2020 ./xmlchange SSTICE_YEAR_END=2021 @@ -16,5 +16,5 @@ ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange $ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/COMBLE_iopfile_4scam.nc -b $ATMCHANGE target_latitude=74.5 -b $ATMCHANGE target_longitude=10.0 -b -$ATMCHANGE iop_coriolis=true +$ATMCHANGE iop_coriolis=true -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands index 8c442acee75b..675448efedc6 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands @@ -1,5 +1,5 @@ # Sets up DPxx case to test DYCOMSrf01 (marine stratocumulus) -# case over ocean using prescribed surface fluxes +# case over ocean using prescribed surface fluxes and compute tendencies of large scale subsidence from prescribed omega ./xmlchange RUN_STARTDATE="1999-07-10" ./xmlchange PTS_LAT=31.5 From 31f5b2934f871bf348c9c3e2ee2cc223ef102596 Mon Sep 17 00:00:00 2001 From: Peter Andrew Bogenschutz Date: Wed, 31 Jul 2024 08:29:08 -0700 Subject: [PATCH 260/477] fix to COMBLE longitude --- .../testdefs/testmods_dirs/scream/dpxx/comble/shell_commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands index 5b3b6989aa41..0cc2f5575bf3 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -3,7 +3,7 @@ ./xmlchange RUN_STARTDATE="2020-03-12" ./xmlchange PTS_LAT=74.5 -./xmlchange PTS_LON=10.0 +./xmlchange PTS_LON=9.9 ./xmlchange SSTICE_DATA_FILENAME='$DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc' ./xmlchange SSTICE_YEAR_ALIGN=2020 @@ -15,6 +15,6 @@ ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange $ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/COMBLE_iopfile_4scam.nc -b $ATMCHANGE target_latitude=74.5 -b -$ATMCHANGE target_longitude=10.0 -b +$ATMCHANGE target_longitude=9.9 -b $ATMCHANGE iop_coriolis=true -b From 408e876005c21fec791b21fe5a8989611b6a50cc Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 31 Jul 2024 10:34:53 -0600 Subject: [PATCH 261/477] Fix a couple of FIXME comments from code review. Added comments containing descriptions and units for locally allocated arrays. Comments only, no chnage in logic. --- ...x_mam_dry_deposition_process_interface.cpp | 31 ++++++--- ...x_mam_dry_deposition_process_interface.hpp | 65 +++++++++++++++++-- .../src/physics/shoc/impl/shoc_main_impl.hpp | 4 +- 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp index 4a9dcbf1eca6..034972f85258 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -318,24 +318,39 @@ void MAMDryDep::initialize_impl(const RunType run_type) { // Allocate memory //----------------------------------------------------------------- const int pcnst = mam4::aero_model::pcnst; - // FIXME: comment what they are and units..... - qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); - rho_ = view_2d("rho", ncol_, nlev_); + // Output of the the mixing ratio tendencies [kg/kg/s or 1/kg/s] ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); - vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), - aerosol_categories_, ncol_, nlev_); - vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), - aerosol_categories_, ncol_, nlev_); + // Deposition velocity of turbulent dry deposition [m/s] vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), aerosol_categories_, ncol_); + // Deposition velocity of gravitational settling [m/s] + vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + // Deposition velocity, [m/s] + // Fraction landuse weighted sum of vlc_grv and vlc_trb + vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + + // Work array to hold the mixing ratios [kg/kg or 1/kg] + // Packs AerosolState::int_aero_nmr and AerosolState::int_aero_nmr + // into one array. + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + + // Work array to hold the air density [kg/m3] + rho_ = view_2d("rho", ncol_, nlev_); + + // Work array to hold cloud borne aerosols mixing ratios [kg/kg or 1/kg] + // Filled with Prognostics::n_mode_c and Prognostics::q_aero_c + qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); - qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); + // Work array to hold tendency for 1 species [kg/kg/s] or [1/kg/s] dqdt_tmp_ = view_3d("dqdt_tmp_", pcnst, ncol_, nlev_); static constexpr int n_land_type = mam4::DryDeposition::n_land_type; // FIXME: This should come from a file reading + // The fraction of land use for the column. [non-dimentional] fraction_landuse_ = view_2d("fraction_landuse_", n_land_type, ncol_); //----------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp index 8feb71e04693..e39090cecdea 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -45,19 +45,70 @@ class MAMDryDep final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - view_3d qtracers_; - view_3d ptend_q_; - // inputs - // FIXME: collect all inputs and outputs together - view_2d fraction_landuse_; + /* Note on mam4::DryDeposition::aerosol_categories = 4 + used in deposition velocity dimension defined below. These + correspond to the two attachment states and two moments: + 0 - interstitial aerosol, 0th moment (i.e., number) + 1 - interstitial aerosol, 3rd moment (i.e., volume/mass) + 2 - cloud-borne aerosol, 0th moment (i.e., number) + 3 - cloud-borne aerosol, 3rd moment (i.e., volume/mass) + see comments in the DryDeposition class in mam4xx. + */ + // Output deposition velocity of turbulent dry deposition [m/s] + // Dimensions + // [numer of modes, aerosol_categories_, num columns] view_3d vlc_trb_; - view_2d rho_; - view_4d vlc_dry_; + // Output deposition velocity of gravitational settling [m/s] + // Dimensions + // [num_modes, aerosol_categories_, num columns, num levels] view_4d vlc_grv_; + + // Output deposition velocity, [m/s] + // fraction landuse weighted sum of vlc_grv and vlc_trb + // Dimensions + // [num_modes, aerosol_categories_, num columns, num levels] + view_4d vlc_dry_; + + + // Output of the the mixing ratio tendencies [kg/kg/s or 1/kg/s] + // Dimensions + // [num columns, num levels, mam4::aero_model::pcnst] + // Packed the same way qtracers_ is layed out. + view_3d ptend_q_; + + // Work array to hold the mixing ratios [kg/kg or 1/kg] + // Dimensions + // [num columns, num levels, mam4::aero_model::pcnst] + // Packs AerosolState::int_aero_nmr + // and AerosolState::int_aero_nmr + // into one array, hence is mixed kg/kg and 1/kg. + view_3d qtracers_; + + // Work array to hold the fraction [non-dimentional] + // of land use for column. + // Dimensions + // [MAMDryDep::n_land_type, num columns] + // Values should sum to 1. + view_2d fraction_landuse_; + + // Work array to hold the air density [kg/m3] + // Dimensions + // [num columns, num levels] + // Calculated from air pressure at layer midpoint, + // Constants::r_gas_dry_air and air temperture. + view_2d rho_; + + // Work array to hold tendency for 1 species [kg/kg/s] or [1/kg/s] + // Dimensions + // [mam4::aero_model::pcnst, num column, num level] view_3d dqdt_tmp_; + // Work array to hold cloud borne aerosols mixing ratios [kg/kg or 1/kg] + // Dimensions + // [mam4::aero_model::pcnst, num column, num level] + // Filled with Prognostics::n_mode_c and Prognostics::q_aero_c view_3d qqcw_; public: diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 807eccdfe559..11383e772454 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -685,9 +685,9 @@ Int Functions::shoc_main( wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables + shoc_output.pblh(i) = pblh_s; + shoc_output.ustar(i) = ustar_s; shoc_output.obklen(i) = obklen_s; - shoc_output.ustar(i) = ustar_s; - shoc_output.pblh(i) = pblh_s; }); Kokkos::fence(); #else From ec1c09b1084fdbc29f935780ed20776724010f5e Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 31 Jul 2024 11:32:41 -0600 Subject: [PATCH 262/477] Update EKAT --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index b39fd9fbd814..9ffec9a4fb4c 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit b39fd9fbd81439470e146d7915ed9a22c688ac6a +Subproject commit 9ffec9a4fb4c03596b7e1729241b6348aad70ec1 From b393388533598f6842992d9a64f7e21ef6000829 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 1 Aug 2024 10:34:38 -0700 Subject: [PATCH 263/477] add correct starting time for COMBLE case --- .../testdefs/testmods_dirs/scream/dpxx/comble/shell_commands | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands index 0cc2f5575bf3..5b5df60cf822 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -2,6 +2,7 @@ # lagrangian case over ocean with interactive surface fluxes and coriolis forcing applied to winds ./xmlchange RUN_STARTDATE="2020-03-12" +./xmlchange START_TOD=79200 ./xmlchange PTS_LAT=74.5 ./xmlchange PTS_LON=9.9 From 89392e4a91afae5dc3cc9370fdc661f637bbe0fb Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 1 Aug 2024 15:24:20 -0600 Subject: [PATCH 264/477] remove multiprocess tests for compute sanitizer --- components/eamxx/tests/CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 5fd23a41d9d8..7602f654356d 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -68,9 +68,13 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) # Testing individual atm processes add_subdirectory(single-process) - # Testing multiple atm processes coupled together - add_subdirectory(multi-process) - + # Testing multiple atm processes coupled together. + # Some compute-sanitizer tests time out with these + # larger multiprocess tests, so disable in that case. + if (NOT EKAT_ENABLE_COMPUTE_SANITIZER) + add_subdirectory(multi-process) + endif() + if (EAMXX_ENABLE_PYBIND) add_subdirectory(python) endif() From b3c5aee66a2d69cc6daf3668011c276d0edd0130 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Thu, 1 Aug 2024 21:08:41 -0700 Subject: [PATCH 265/477] Set default cloud condensate in IC and increase tolerance for difference in atm/lnd lat/lon in domain checking --- components/eamxx/cime_config/namelist_defaults_scream.xml | 8 ++++---- driver-mct/cime_config/namelist_definition_drv.xml | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50188236e4e0..2714c34d9c20 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -517,10 +517,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 - 0.0 - 0.0 - 0.0 - 0.0 + 0.0 + 0.0 + 0.0 + 0.0 0.0 diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index 7fbf83688c8a..982acabe64d8 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -1527,6 +1527,7 @@ 1.e-10 1.e-10 1.e-10 + 1.e-10 From fc26235f0a0ac961111d83e0db1239cec55a554c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 06:43:52 -0700 Subject: [PATCH 266/477] Adds a CIME drydep test --- cime_config/tests.py | 1 + .../scream/mam4xx/drydep/shell_commands | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index b4fb91cc5ee1..4750de448765 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -725,6 +725,7 @@ "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-drydep", ) }, diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands index d219b2a489e0..c5acf055a882 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands @@ -1,12 +1,16 @@ -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ -#modify initial condition file to get aerosol species ICs -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b -# Add drydep process +#------------------------------------------------------ +#Update IC file and add drydep process +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_drydep" -b From f0fc790936de3d22f0a412ddc7a5cbd73797eff6 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Fri, 2 Aug 2024 14:04:44 -0700 Subject: [PATCH 267/477] Update Livermore Computing machine config files Replaces gdata with workspace for Livermore Machines Removes quartz machine Adds dane machine [BFB] --- .../{intel_quartz.cmake => intel_dane.cmake} | 0 cime_config/machines/config_machines.xml | 34 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) rename cime_config/machines/cmake_macros/{intel_quartz.cmake => intel_dane.cmake} (100%) diff --git a/cime_config/machines/cmake_macros/intel_quartz.cmake b/cime_config/machines/cmake_macros/intel_dane.cmake similarity index 100% rename from cime_config/machines/cmake_macros/intel_quartz.cmake rename to cime_config/machines/cmake_macros/intel_dane.cmake diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7cd34fdb0247..f821ff66ba4f 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2978,14 +2978,14 @@ mpich cbronze /p/lustre2/$USER/e3sm_scratch/ruby - /usr/gdata/e3sm/ccsm3data/inputdata - /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 + /usr/workspace/e3sm/ccsm3data/inputdata + /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/e3sm/tools/cprnc + /usr/workspace/e3sm/tools/cprnc 8 lc_slurm - donahue5 -at- llnl.gov + boutte3 -at- llnl.gov 56 56 @@ -3010,7 +3010,7 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/e3sm/install/quartz/modulefiles + /usr/workspace/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 @@ -3022,28 +3022,28 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/e3sm/install/quartz/netcdf-fortran/ + /usr/workspace/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 - - LLNL Linux Cluster, Linux (pgi), 36 pes/node, batch system is Slurm + + LLNL Linux Cluster, 112 pes/node, batch system is Slurm LINUX intel mpich cbronze - /p/lustre2/$USER/e3sm_scratch/quartz - /usr/gdata/e3sm/ccsm3data/inputdata - /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 + /p/lustre2/$USER/e3sm_scratch/dane + /usr/workspace/e3sm/ccsm3data/inputdata + /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/e3sm/tools/cprnc + /usr/workspace/e3sm/tools/cprnc 8 lc_slurm - donahue5 -at- llnl.gov - 72 - 36 + boutte3 -at- llnl.gov + 224 + 112 @@ -3066,7 +3066,7 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/e3sm/install/quartz/modulefiles + /usr/workspace/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 @@ -3078,7 +3078,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/e3sm/install/quartz/netcdf-fortran/ + /usr/workspace/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 From faa559bf8ca778bc6c0738d8994bc56236fcf599 Mon Sep 17 00:00:00 2001 From: dqwu Date: Sun, 4 Aug 2024 11:35:49 -0500 Subject: [PATCH 268/477] Use cpe/22.12 instead of PrgEnv-cray to undo unintended changes On Frontier, the "one module to rule them all" is typically cpe. Changing cpe versions alters defaults for most other modules. The change in the default cpe version (22.12 -> 23.12) may be one of the main breaking changes for post-maintenance SCREAM builds. Trey White recommended using cpe/22.12 to get as close as possible to the pre-maintenance configuration. This also allows us to use libunwind/1.6.2 without loading the ums modules. --- cime_config/machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index f821ff66ba4f..eeb80756bb8c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1434,10 +1434,10 @@ /usr/share/lmod/lmod/libexec/lmod python - PrgEnv-cray + cpe/22.12 craype-accel-amd-gfx90a rocm/5.4.0 - libunwind/1.5.0 + libunwind/1.6.2 cce/15.0.1 From 0830f8d736f8cb91c6517b95cb3559d17147aa82 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 5 Aug 2024 18:24:12 -0700 Subject: [PATCH 269/477] change DPxx compset alias names and rename General compset to IOP --- cime_config/tests.py | 8 ++++---- components/eamxx/cime_config/config_compsets.xml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index dd08de8c2540..ac999e0369cf 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -657,10 +657,10 @@ "time" : "01:00:00", # each test 225 phys cols, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-dycomsrf01", - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-arm97", - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-General.scream-dpxx-comble", - "ERS_P16_Ln22.ne30_ne30.F2000-SCREAMv1-DP-RCE", + "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-arm97", + "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-comble", + "ERS_P16_Ln22.ne30_ne30.FRCE-SCREAMv1-DP", ) }, diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index c684e1f6c6b6..e23548d03708 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -75,13 +75,13 @@ - F2010-SCREAMv1-DP-General - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%General + FIOP-SCREAMv1-DP + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%IOP Experimental, under development - F2000-SCREAMv1-DP-RCE + FRCE-SCREAMv1-DP 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx Experimental, under development From 23dc534c2ed731fbcd2f6fa85b9158622493c62c Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 6 Aug 2024 10:10:42 -0700 Subject: [PATCH 270/477] run DPxx tests with PG2 grid --- cime_config/tests.py | 8 ++++---- components/eamxx/cime_config/config_compsets.xml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1ad9844e2e9e..1418529e78b4 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -657,10 +657,10 @@ "time" : "01:00:00", # each test 225 phys cols, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", - "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-arm97", - "ERS_P16_Ln22.ne30_ne30.FIOP-SCREAMv1-DP.scream-dpxx-comble", - "ERS_P16_Ln22.ne30_ne30.FRCE-SCREAMv1-DP", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-comble", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FRCE-SCREAMv1-DP", ) }, diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index e23548d03708..b378efcfdebe 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -117,7 +117,7 @@ - 225 + 100 @@ -129,7 +129,7 @@ - 225 + 100 From 946bc5448e11ad1a2628aaacfbd0db048afec47c Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 6 Aug 2024 10:34:26 -0700 Subject: [PATCH 271/477] update comment pertaining to number of columns in tests --- cime_config/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 1418529e78b4..a3a5949f8435 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -654,8 +654,8 @@ }, "e3sm_scream_v1_dp-eamxx" : { - "time" : "01:00:00", - # each test 225 phys cols, roughly size of ne2 + "time" : "00:30:00", + # each test runs with 225 dynamics and 100 physics columns, roughly size of ne2 "tests" : ( "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97", From bcc1ba1b1dc9fa5ada1e32d7477b4c148e1238a2 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Tue, 6 Aug 2024 10:35:32 -0700 Subject: [PATCH 272/477] undo time change in DPxx tests --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index a3a5949f8435..8055dd261b29 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -654,7 +654,7 @@ }, "e3sm_scream_v1_dp-eamxx" : { - "time" : "00:30:00", + "time" : "01:00:00", # each test runs with 225 dynamics and 100 physics columns, roughly size of ne2 "tests" : ( "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", From 5e5d66ec392526bf88614f638ea151eb86a5ac38 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 6 Aug 2024 12:31:55 -0600 Subject: [PATCH 273/477] EAMxx: fix usage of ParameterList in one of the unit tests --- components/eamxx/src/share/tests/atm_process_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/tests/atm_process_tests.cpp b/components/eamxx/src/share/tests/atm_process_tests.cpp index 01985c33c6d0..fce25cd28e21 100644 --- a/components/eamxx/src/share/tests/atm_process_tests.cpp +++ b/components/eamxx/src/share/tests/atm_process_tests.cpp @@ -443,7 +443,7 @@ TEST_CASE("atm_proc_dag", "") { using strvec_t = std::vector; auto params = create_test_params(); - auto p1 = params.sublist("BarBaz"); + auto& p1 = params.sublist("BarBaz"); // Make sure there's a missing piece (whatever Baz computes); p1.set("atm_procs_list",{"Bar"}); From d65256783f6db64a24bebe7779aadaca66cd83db Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Wed, 17 Jul 2024 09:02:50 -0600 Subject: [PATCH 274/477] Port shoc_assumed_pdf functions individually (like in shoc.F90) --- .../eamxx/src/physics/shoc/CMakeLists.txt | 12 + ...shoc_assumed_pdf_compute_buoyancy_flux.cpp | 14 + ...umed_pdf_compute_cloud_liquid_variance.cpp | 14 + ..._assumed_pdf_compute_liquid_water_flux.cpp | 14 + .../shoc/eti/shoc_assumed_pdf_compute_qs.cpp | 14 + .../shoc/eti/shoc_assumed_pdf_compute_s.cpp | 14 + .../shoc_assumed_pdf_compute_sgs_liquid.cpp | 14 + .../shoc_assumed_pdf_compute_temperature.cpp | 14 + .../shoc_assumed_pdf_inplume_correlations.cpp | 14 + .../eti/shoc_assumed_pdf_qw_parameters.cpp | 14 + .../eti/shoc_assumed_pdf_thl_parameters.cpp | 14 + .../eti/shoc_assumed_pdf_tilde_to_real.cpp | 14 + .../eti/shoc_assumed_pdf_vv_parameters.cpp | 14 + ...assumed_pdf_compute_buoyancy_flux_impl.hpp | 41 +++ ...pdf_compute_cloud_liquid_variance_impl.hpp | 39 +++ ...med_pdf_compute_liquid_water_flux_impl.hpp | 33 +++ .../impl/shoc_assumed_pdf_compute_qs_impl.hpp | 55 ++++ .../impl/shoc_assumed_pdf_compute_s_impl.hpp | 69 +++++ ...oc_assumed_pdf_compute_sgs_liquid_impl.hpp | 30 ++ ...c_assumed_pdf_compute_temperature_impl.hpp | 29 ++ .../shoc/impl/shoc_assumed_pdf_impl.hpp | 264 ++++-------------- ..._assumed_pdf_inplume_correlations_impl.hpp | 50 ++++ .../shoc_assumed_pdf_qw_parameters_impl.hpp | 79 ++++++ .../shoc_assumed_pdf_thl_parameters_impl.hpp | 82 ++++++ .../shoc_assumed_pdf_tilde_to_real_impl.hpp | 32 +++ .../shoc_assumed_pdf_vv_parameters_impl.hpp | 61 ++++ .../eamxx/src/physics/shoc/shoc_functions.hpp | 164 ++++++++++- 27 files changed, 992 insertions(+), 216 deletions(-) create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp create mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp create mode 100644 components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 891626b9d3df..e6724622a928 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -25,6 +25,18 @@ if (NOT EAMXX_ENABLE_GPU) list(APPEND SHOC_SRCS eti/shoc_adv_sgs_tke.cpp eti/shoc_assumed_pdf.cpp + eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp + eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp + eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp + eti/shoc_assumed_pdf_compute_qs.cpp + eti/shoc_assumed_pdf_compute_sgs_liquid.cpp + eti/shoc_assumed_pdf_compute_temperature.cpp + eti/shoc_assumed_pdf_inplume_correlations.cpp + eti/shoc_assumed_pdf_qw_parameters.cpp + eti/shoc_assumed_pdf_compute_s.cpp + eti/shoc_assumed_pdf_thl_parameters.cpp + eti/shoc_assumed_pdf_tilde_to_real.cpp + eti/shoc_assumed_pdf_vv_parameters.cpp eti/shoc_calc_shoc_varorcovar.cpp eti/shoc_calc_shoc_vertflux.cpp eti/shoc_check_length_scale_shoc_length.cpp diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp new file mode 100644 index 000000000000..4d38b8525dd8 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_buoyancy_flux on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp new file mode 100644 index 000000000000..eb83e2098772 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_cloud_liquid_variance on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp new file mode 100644 index 000000000000..d8f99806cf9b --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_liquid_water_flux on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp new file mode 100644 index 000000000000..fbac9733184b --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_qs_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_qs on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp new file mode 100644 index 000000000000..b74ae269a764 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_s_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_s on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp new file mode 100644 index 000000000000..d2c786e88368 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_sgs_liquid on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp new file mode 100644 index 000000000000..8fdf36d64088 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_compute_temperature_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_compute_temperature on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp new file mode 100644 index 000000000000..d29641dd73bf --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_inplume_correlations_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_inplume_correlations on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp new file mode 100644 index 000000000000..b9a9501b9887 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_qw_parameters_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_qw_parameters on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp new file mode 100644 index 000000000000..716fd7cd3d29 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_thl_parameters_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_thl_parameters on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp new file mode 100644 index 000000000000..250e3f1a0a40 --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_tilde_to_real_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_tilde_to_real on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp new file mode 100644 index 000000000000..2af26cfc1adb --- /dev/null +++ b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp @@ -0,0 +1,14 @@ +#include "shoc_assumed_pdf_vv_parameters_impl.hpp" + +namespace scream { +namespace shoc { + +/* + * Explicit instantiation for doing shoc_assumed_pdf_vv_parameters on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp new file mode 100644 index 000000000000..db31f1ab1bd5 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp @@ -0,0 +1,41 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_BUOYANCY_FLUX_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_BUOYANCY_FLUX_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_buoyancy_flux. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_buoyancy_flux( + const Spack& wthlsec, + const Spack& wqwsec, + const Spack& pval, + const Spack& wqls, + Spack& wthv_sec) +{ + const Scalar basepres = C::P0; + const Scalar rair = C::Rair; + const Scalar rv = C::RV; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + const Scalar basetemp = C::basetemp; + const Scalar epsterm = rair/rv; + + wthv_sec = wthlsec + ((1 - epsterm)/epsterm)*basetemp*wqwsec + + ((lcond/cp)*ekat::pow(basepres/pval, (rair/cp)) + - (1/epsterm)*basetemp)*wqls; +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp new file mode 100644 index 000000000000..06f740d6eb55 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp @@ -0,0 +1,39 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_CLOUD_LIQUID_VARIANCE_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_CLOUD_LIQUID_VARIANCE_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_cloud_liquid_variance. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_cloud_liquid_variance( + const Spack& a, + const Spack& s1, + const Spack& ql1, + const Spack& C1, + const Spack& std_s1, + const Spack& s2, + const Spack& ql2, + const Spack& C2, + const Spack& std_s2, + const Spack& shoc_ql, + Spack& shoc_ql2) +{ + shoc_ql2 = ekat::max(0, a*(s1*ql1 + C1*ekat::square(std_s1)) + + (1 - a)*(s2*ql2 + C2*ekat::square(std_s2)) + - ekat::square(shoc_ql)); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp new file mode 100644 index 000000000000..b225494e7b6d --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp @@ -0,0 +1,33 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_LIQUID_WATER_FLUX_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_LIQUID_WATER_FLUX_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_liquid_water_flux. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_liquid_water_flux( + const Spack& a, + const Spack& w1_1, + const Spack& w_first, + const Spack& ql1, + const Spack& w1_2, + const Spack& ql2, + Spack& wqls) +{ + wqls = a*((w1_1 - w_first)*ql1) + (1 - a)*((w1_2 - w_first)*ql2); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp new file mode 100644 index 000000000000..3949f087bfe2 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp @@ -0,0 +1,55 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_QS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_QS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU +#include "physics_functions.hpp" + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_qs. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_qs( + const Spack& Tl1_1, + const Spack& Tl1_2, + const Spack& pval, + const Smask& active_entries, + Spack& qs1, + Spack& beta1, + Spack& qs2, + Spack& beta2) +{ + const Scalar rair = C::Rair; + const Scalar rv = C::RV; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + + // Compute MurphyKoop_svp + const int liquid = 0; + const Spack esval1_1 = scream::physics::Functions::MurphyKoop_svp(Tl1_1,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_1)"); + const Spack esval1_2 = scream::physics::Functions::MurphyKoop_svp(Tl1_2,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_2)"); + const Spack lstarn(lcond); + + qs1 = sp(0.622)*esval1_1/ekat::max(esval1_1, pval - esval1_1); + beta1 = (rair/rv)*(lstarn/(rair*Tl1_1))*(lstarn/(cp*Tl1_1)); + + // Only compute qs2 and beta2 if the two plumes are not equal + const Smask condition = (Tl1_1 != Tl1_2); + qs2 = qs1; + beta2 = beta1; + + qs2.set(condition, sp(0.622)*esval1_2/ekat::max(esval1_2, pval - esval1_2)); + beta2.set(condition, (rair/rv)*(lstarn/(rair*Tl1_2))*(lstarn/(cp*Tl1_2))); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp new file mode 100644 index 000000000000..180bf889d362 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp @@ -0,0 +1,69 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_S_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_S_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_s. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_s( + const Spack& qw1, + const Spack& qs, + const Spack& beta, + const Spack& pval, + const Spack& thl2, + const Spack& qw2, + const Spack& sqrtthl2, + const Spack& sqrtqw2, + const Spack& r_qwthl, + Spack& s, + Spack& std_s, + Spack& qn, + Spack& C) +{ + const Scalar rair = C::Rair; + const Scalar basepres = C::P0; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + const Scalar pi = C::Pi; + + const Scalar sqrt2(std::sqrt(Scalar(2.0))), sqrt2pi(std::sqrt(2*pi)); + + const Spack cthl=((1 + beta*qw1)/ekat::square(1 + beta*qs))*(cp/lcond)* + beta*qs*ekat::pow(pval/basepres, (rair/cp)); + const Spack cqt = 1/(1 + beta*qs); + + std_s = ekat::sqrt(ekat::max(0, + ekat::square(cthl)*thl2 + + ekat::square(cqt)*qw2 - 2*cthl*sqrtthl2*cqt*sqrtqw2*r_qwthl)); + const auto std_s_not_small = std_s > std::sqrt(std::numeric_limits::min()) * 100; + s = qw1-qs*((1 + beta*qw1)/(1 + beta*qs)); + if (std_s_not_small.any()) { + C.set(std_s_not_small, sp(0.5)*(1 + ekat::erf(s/(sqrt2*std_s)))); + } + C.set(!std_s_not_small && s > 0, 1); + const auto std_s_C_not_small = std_s_not_small && C != 0; + if (std_s_C_not_small.any()) { + qn.set(std_s_C_not_small, s*C+(std_s/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s/std_s))); + } + qn.set(!std_s_not_small && s > 0, s); + + // Checking to prevent empty clouds + const auto qn_le_zero = qn <= 0; + C.set(qn_le_zero,0); + qn.set(qn_le_zero,0); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp new file mode 100644 index 000000000000..cbc12f99a673 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp @@ -0,0 +1,30 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_SGS_LIQUID_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_SGS_LIQUID_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_sgs_liquid. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_sgs_liquid( + const Spack& a, + const Spack& ql1, + const Spack& ql2, + Spack& shoc_ql) +{ + shoc_ql = ekat::max(0, a*ql1 + (1 - a)*ql2); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp new file mode 100644 index 000000000000..e857951c5f2b --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp @@ -0,0 +1,29 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_TEMPERATURE_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_TEMPERATURE_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_temperature. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_compute_temperature( + const Spack& thl1, + const Spack& pval, + Spack& Tl1) +{ + Tl1 = thl1/(ekat::pow(C::P0/pval,(C::Rair/C::CP))); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp index 84e76c5941a1..cd004adbd2d6 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp @@ -77,15 +77,6 @@ void Functions::shoc_assumed_pdf( const Scalar w_tol_sqd = 4e-4; const Scalar w_thresh = 0; const Scalar largeneg = SC::largeneg; - const Scalar rair = C::Rair; - const Scalar rv = C::RV; - const bool dothetal_skew = SC::dothetal_skew; - const Scalar basepres = C::P0; - const Scalar cp = C::CP; - const Scalar lcond = C::LatVap; - const Scalar pi = C::Pi; - const Scalar basetemp = C::basetemp; - const Scalar epsterm = rair/rv; const Scalar Tl_min = 100; // Interpolate many variables from interface grid to thermo grid @@ -125,121 +116,35 @@ void Functions::shoc_assumed_pdf( const auto sqrtqt = ekat::max(rt_tol,ekat::sqrt(qwsec)); // Find parameters for vertical velocity - Spack Skew_w(0), w1_1(w_first), w1_2(w_first), w2_1(0), w2_2(0), a(0.5); - { - const Smask condition = w_sec(k) > w_tol_sqd; - - const Scalar tmp_val(0.4); - const Scalar one_m_tmp_val(1 - tmp_val); - const Scalar sqrtw2t(std::sqrt(1-tmp_val)); - - Skew_w.set(condition, w3var/ekat::sqrt(ekat::cube(w_sec(k)))); - a.set(condition, - ekat::max(sp(0.01), - ekat::min(sp(0.99), - sp(0.5)*(1 - Skew_w*ekat::sqrt(1/(4*(one_m_tmp_val*one_m_tmp_val*one_m_tmp_val) - + ekat::square(Skew_w))))))); - - w1_1.set(condition, ekat::sqrt((1 - a)/a)*sqrtw2t); - w1_2.set(condition, -1*ekat::sqrt(a/(1 - a))*sqrtw2t); - w2_1.set(condition, tmp_val*w_sec(k)); - w2_2.set(condition, tmp_val*w_sec(k)); - } + Spack Skew_w, w1_1, w1_2, w2_1, w2_2, a; + shoc_assumed_pdf_vv_parameters(w_first, w2sec, w3var, w_tol_sqd, Skew_w, w1_1, w1_2, w2_1, w2_2, a); // Find parameters for thetal - Spack thl1_1(thl_first), thl1_2(thl_first), thl2_1(0), thl2_2(0), - sqrtthl2_1(0), sqrtthl2_2(0); - { - const Smask condition = thlsec > (thl_tol*thl_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; - - const Spack corrtest1 = ekat::max(-1, ekat::min(1, wthlsec/(sqrtw2*sqrtthl))); - const Spack tmp_val_1(-corrtest1/w1_1), tmp_val_2(-corrtest1/w1_2); - - Spack Skew_thl(0); - if (dothetal_skew == true) { - const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); - Skew_thl.set(tsign>sp(0.4), sp(1.2)*Skew_w); - Skew_thl.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); - } - - if (condition.any()) { - thl2_1.set(condition, - ekat::min(100, - ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1-a)*ekat::square(tmp_val_1)) - - (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*a*(tmp_val_1 - tmp_val_2))))*thlsec); - thl2_2.set(condition, - ekat::min(100, - ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - - (1 - a)*ekat::square(tmp_val_1)) - + (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*thlsec); - - thl1_1.set(condition, tmp_val_2*sqrtthl+thl_first); - thl1_2.set(condition, tmp_val_1*sqrtthl+thl_first); - - sqrtthl2_1.set(condition, ekat::sqrt(thl2_1)); - sqrtthl2_2.set(condition, ekat::sqrt(thl2_2)); - } - - } + Spack thl1_1, thl1_2, thl2_1, thl2_2, sqrtthl2_1, sqrtthl2_2; + shoc_assumed_pdf_thl_parameters(wthlsec,sqrtw2,sqrtthl,thlsec,thl_first,w1_1,w1_2,Skew_w,a, + thl_tol, w_thresh, + thl1_1, thl1_2, thl2_1, thl2_2, sqrtthl2_1, sqrtthl2_2); // Find parameters for total water mixing ratio - Spack qw1_1(qw_first), qw1_2(qw_first), qw2_1(0), qw2_2(0), - sqrtqw2_1(0), sqrtqw2_2(0); - { - const Smask condition = qwsec > (rt_tol*rt_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; - - const Spack corrtest2 = ekat::max(-1, ekat::min(1, wqwsec/(sqrtw2*sqrtqt))); - const Spack tmp_val_1(-corrtest2/w1_1), tmp_val_2(-corrtest2/w1_2); - - const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); - Spack Skew_qw(0); - Skew_qw.set(tsign>sp(0.4), sp(1.2)*Skew_w); - Skew_qw.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); - - if (condition.any()) { - qw2_1.set(condition, - ekat::min(100, - ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) - - (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*a*(tmp_val_1 - tmp_val_2))))*qwsec); - qw2_2.set(condition, - ekat::min(100, - ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) - + (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*qwsec); - } - - qw1_1.set(condition, tmp_val_2*sqrtqt+qw_first); - qw1_2.set(condition, tmp_val_1*sqrtqt+qw_first); - - sqrtqw2_1.set(condition, ekat::sqrt(qw2_1)); - sqrtqw2_2.set(condition, ekat::sqrt(qw2_2)); - } + Spack qw1_1, qw1_2, qw2_1, qw2_2, sqrtqw2_1, sqrtqw2_2; + shoc_assumed_pdf_qw_parameters(wqwsec, sqrtw2, Skew_w, sqrtqt, qwsec, w1_2, w1_1, qw_first, a, + rt_tol, w_thresh, + qw1_1, qw1_2, qw2_1, qw2_2, sqrtqw2_1, sqrtqw2_2); // Convert from tilde variables to "real" variables - w1_1 *= sqrtw2; - w1_1 += w_first; - w1_2 *= sqrtw2; - w1_2 += w_first; + shoc_assumed_pdf_tilde_to_real(w_first, sqrtw2, w1_1); + shoc_assumed_pdf_tilde_to_real(w_first, sqrtw2, w1_2); // Find within-plume correlations. - Spack r_qwthl_1(0); - { - const Spack testvar = a*sqrtqw2_1*sqrtthl2_1 + (1 - a)*sqrtqw2_2*sqrtthl2_2; - const auto testvar_ne_zero = testvar != 0; - if (testvar_ne_zero.any()) { - r_qwthl_1.set(testvar_ne_zero, - ekat::max(-1, - ekat::min(1, (qwthlsec - a*(qw1_1 - qw_first)*(thl1_1 - thl_first) - - (1 - a)*(qw1_2 - qw_first)*(thl1_2 - thl_first))/testvar))); - } - } + Spack r_qwthl_1; + shoc_assumed_pdf_inplume_correlations(sqrtqw2_1, sqrtthl2_1, a, sqrtqw2_2, sqrtthl2_2, qwthlsec, + qw1_1, qw_first, thl1_1, thl_first, qw1_2, thl1_2, + r_qwthl_1); // Begin to compute cloud property statistics - Spack Tl1_1 = thl1_1/(ekat::pow(basepres/pval,(rair/cp))); - Spack Tl1_2 = thl1_2/(ekat::pow(basepres/pval,(rair/cp))); + Spack Tl1_1, Tl1_2; + shoc_assumed_pdf_compute_temperature(thl1_1,pval, Tl1_1); + shoc_assumed_pdf_compute_temperature(thl1_2,pval, Tl1_2); const auto index_range = ekat::range(k*Spack::n); const Smask active_entries = (index_range < nlev); @@ -295,121 +200,54 @@ void Functions::shoc_assumed_pdf( } // Compute qs and beta - Spack qs1(0), qs2(0), beta1(0), beta2(0); - { - // Compute MurphyKoop_svp - const int liquid = 0; - const Spack esval1_1 = scream::physics::Functions::MurphyKoop_svp(Tl1_1,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_1)"); - const Spack esval1_2 = scream::physics::Functions::MurphyKoop_svp(Tl1_2,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_2)"); - const Spack lstarn(lcond); - - qs1 = sp(0.622)*esval1_1/ekat::max(esval1_1, pval - esval1_1); - beta1 = (rair/rv)*(lstarn/(rair*Tl1_1))*(lstarn/(cp*Tl1_1)); - - // Only compute qs2 and beta2 if the two plumes are not equal - const Smask condition = (Tl1_1 != Tl1_2); - qs2 = qs1; - beta2 = beta1; - - qs2.set(condition, sp(0.622)*esval1_2/ekat::max(esval1_2, pval - esval1_2)); - beta2.set(condition, (rair/rv)*(lstarn/(rair*Tl1_2))*(lstarn/(cp*Tl1_2))); - } + Spack qs1, qs2, beta1, beta2; + shoc_assumed_pdf_compute_qs(Tl1_1, Tl1_2, pval, active_entries, qs1, beta1, qs2, beta2); // Cloud computations // Compute s terms. - Spack s1(0), std_s1(0), qn1(0), C1(0), ql1(0), - s2(0), std_s2(0), qn2(0), C2(0), ql2(0); - { - const Scalar sqrt2(std::sqrt(Scalar(2.0))), sqrt2pi(std::sqrt(2*pi)); - - // First plume - const Spack cthl1=((1 + beta1*qw1_1)/ekat::square(1 + beta1*qs1))*(cp/lcond)* - beta1*qs1*ekat::pow(pval/basepres, (rair/cp)); - const Spack cqt1 = 1/(1 + beta1*qs1); - - std_s1 = ekat::sqrt(ekat::max(0, - ekat::square(cthl1)*thl2_1 - + ekat::square(cqt1)*qw2_1 - 2*cthl1*sqrtthl2_1*cqt1*sqrtqw2_1*r_qwthl_1)); - const auto std_s1_not_small = std_s1 > std::sqrt(safe_min()) * 100; - s1 = qw1_1-qs1*((1 + beta1*qw1_1)/(1 + beta1*qs1)); - if (std_s1_not_small.any()) { - C1.set(std_s1_not_small, sp(0.5)*(1 + ekat::erf(s1/(sqrt2*std_s1)))); - } - C1.set(!std_s1_not_small && s1 > 0, 1); - const auto std_s1_C1_not_small = std_s1_not_small && C1 != 0; - if (std_s1_C1_not_small.any()) { - qn1.set(std_s1_C1_not_small, s1*C1+(std_s1/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s1/std_s1))); - } - qn1.set(!std_s1_not_small && s1 > 0, s1); - - // Checking to prevent empty clouds - const auto qn1_le_zero = qn1 <= 0; - C1.set(qn1_le_zero,0); - qn1.set(qn1_le_zero,0); - - ql1 = ekat::min(qn1, qw1_1); - - // Second plume - // Only compute variables of the second plume if the two plumes are not equal - const Smask equal(qw1_1==qw1_2 && thl2_1==thl2_2 && qs1==qs2); - std_s2.set(equal, std_s1); - s2.set(equal, s1); - C2.set(equal, C1); - qn2.set(equal, qn1); - - const Spack cthl2(!equal, - ((1 + beta2*qw1_2)/ekat::square(1 + beta2*qs2))*(cp/lcond)* - beta2*qs2*ekat::pow(pval/basepres, (rair/cp))); - const Spack cqt2(!equal, - 1/(1 + beta2*qs2)); - - const auto nequal = !equal; - if (nequal.any()) { - std_s2.set(nequal, - ekat::sqrt(ekat::max(0, - ekat::square(cthl2)*thl2_2 - + ekat::square(cqt2)*qw2_2 - 2*cthl2*sqrtthl2_2*cqt2*sqrtqw2_2*r_qwthl_1))); - s2.set(nequal, qw1_2-qs2*((1 + beta2*qw1_2)/(1 + beta2*qs2))); - const auto std_s2_not_small = std_s2 > std::sqrt(safe_min()) * 100; - const auto nequal_std_s2_not_small = nequal && std_s2_not_small; - if (nequal_std_s2_not_small.any()) { - C2.set(nequal_std_s2_not_small, sp(0.5)*(1 + ekat::erf(s2/(sqrt2*std_s2)))); - } - C2.set(nequal && !std_s2_not_small && s2 > 0, 1); - const auto nequal_std_s2_C2_not_small = nequal_std_s2_not_small && C2 != 0; - if (nequal_std_s2_C2_not_small.any()) { - qn2.set(nequal_std_s2_C2_not_small, s2*C2+(std_s2/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s2/std_s2))); - } - qn2.set(nequal && !std_s2_not_small && s2 > 0, s2); - } - - // Checking to prevent empty clouds - const auto qn2_le_zero = qn2 <= 0; - C2.set(qn2_le_zero,0); - qn2.set(qn2_le_zero,0); - - ql2 = ekat::min(qn2, qw1_2); + Spack s1, std_s1, qn1, C1, ql1, s2, std_s2, qn2, C2, ql2; + + // First plume + shoc_assumed_pdf_compute_s(qw1_1, qs1, beta1, pval, thl2_1, qw2_1, + sqrtthl2_1, sqrtqw2_1, r_qwthl_1, + s1, std_s1, qn1, C1); + + // Second plume + // Only compute variables of the second plume if the two plumes are not equal + const Smask equal(qw1_1==qw1_2 && thl2_1==thl2_2 && qs1==qs2); + std_s2.set(equal, std_s1); + s2.set(equal, s1); + C2.set(equal, C1); + qn2.set(equal, qn1); + + const auto nequal = !equal; + if (nequal.any()) { + shoc_assumed_pdf_compute_s(qw1_2, qs2, beta2, pval, thl2_2, qw2_2, + sqrtthl2_2, sqrtqw2_2, r_qwthl_1, + s2, std_s2, qn2, C2); } + ql1 = ekat::min(qn1, qw1_1); + ql2 = ekat::min(qn2, qw1_2); + // Compute SGS cloud fraction shoc_cldfrac(k) = ekat::min(1, a*C1 + (1 - a)*C2); // Compute SGS liquid water mixing ratio - shoc_ql(k) = ekat::max(0, a*ql1 + (1 - a)*ql2); + shoc_assumed_pdf_compute_sgs_liquid(a, ql1, ql2, shoc_ql(k)); // Compute cloud liquid variance (CLUBB formulation, adjusted to SHOC parameters based) - shoc_ql2(k) = ekat::max(0, a*(s1*ql1 + C1*ekat::square(std_s1)) - + (1 - a)*(s2*ql2 + C2*ekat::square(std_s2)) - - ekat::square(shoc_ql(k))); + shoc_assumed_pdf_compute_cloud_liquid_variance(a, s1, ql1, C1, std_s1, + s2, ql2, C2, std_s2, shoc_ql(k), + shoc_ql2(k)); // Compute liquid water flux - wqls(k) = a*((w1_1 - w_first)*ql1) + (1 - a)*((w1_2 - w_first)*ql2); + shoc_assumed_pdf_compute_liquid_water_flux(a, w1_1, w_first, ql1, w1_2, ql2, wqls(k)); // Compute the SGS buoyancy flux - wthv_sec(k) = wthlsec + ((1 - epsterm)/epsterm)*basetemp*wqwsec - + ((lcond/cp)*ekat::pow(basepres/pval, (rair/cp)) - - (1/epsterm)*basetemp)*wqls(k); + shoc_assumed_pdf_compute_buoyancy_flux(wthlsec, wqwsec, pval, wqls(k), + wthv_sec(k)); }); // Release temporary variables from the workspace diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp new file mode 100644 index 000000000000..52554ea9c98a --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp @@ -0,0 +1,50 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_INPLUME_CORRELATIONS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_INPLUME_CORRELATIONS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_inplume_correlations. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find within-plume correlation + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_inplume_correlations( + const Spack& sqrtqw2_1, + const Spack& sqrtthl2_1, + const Spack& a, + const Spack& sqrtqw2_2, + const Spack& sqrtthl2_2, + const Spack& qwthlsec, + const Spack& qw1_1, + const Spack& qw_first, + const Spack& thl1_1, + const Spack& thl_first, + const Spack& qw1_2, + const Spack& thl1_2, + Spack& r_qwthl_1) +{ + r_qwthl_1 = 0; + + const Spack testvar = a*sqrtqw2_1*sqrtthl2_1 + (1 - a)*sqrtqw2_2*sqrtthl2_2; + const auto testvar_ne_zero = testvar != 0; + if (testvar_ne_zero.any()) { + r_qwthl_1.set(testvar_ne_zero, + ekat::max(-1, + ekat::min(1, (qwthlsec - a*(qw1_1 - qw_first)*(thl1_1 - thl_first) + - (1 - a)*(qw1_2 - qw_first)*(thl1_2 - thl_first))/testvar))); + } +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp new file mode 100644 index 000000000000..546be97f6fb4 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp @@ -0,0 +1,79 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_QW_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_QW_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_qw_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for total water mixing ratio + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_qw_parameters( + const Spack& wqwsec, + const Spack& sqrtw2, + const Spack& Skew_w, + const Spack& sqrtqt, + const Spack& qwsec, + const Spack& w1_2, + const Spack& w1_1, + const Spack& qw_first, + const Spack& a, + const Scalar rt_tol, + const Scalar w_thresh, + Spack& qw1_1, + Spack& qw1_2, + Spack& qw2_1, + Spack& qw2_2, + Spack& sqrtqw2_1, + Spack& sqrtqw2_2) +{ + qw1_1 = qw_first; + qw1_2 = qw_first; + qw2_1 = 0; + qw2_2 = 0; + sqrtqw2_1 = 0; + sqrtqw2_2 = 0; + + const Smask condition = qwsec > (rt_tol*rt_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; + + const Spack corrtest2 = ekat::max(-1, ekat::min(1, wqwsec/(sqrtw2*sqrtqt))); + const Spack tmp_val_1(-corrtest2/w1_1), tmp_val_2(-corrtest2/w1_2); + + const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); + Spack Skew_qw(0); + Skew_qw.set(tsign>sp(0.4), sp(1.2)*Skew_w); + Skew_qw.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); + + if (condition.any()) { + qw2_1.set(condition, + ekat::min(100, + ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) + - (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*a*(tmp_val_1 - tmp_val_2))))*qwsec); + qw2_2.set(condition, + ekat::min(100, + ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) + + (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*qwsec); + } + + qw1_1.set(condition, tmp_val_2*sqrtqt+qw_first); + qw1_2.set(condition, tmp_val_1*sqrtqt+qw_first); + + sqrtqw2_1.set(condition, ekat::sqrt(qw2_1)); + sqrtqw2_2.set(condition, ekat::sqrt(qw2_2)); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp new file mode 100644 index 000000000000..90d60e439b56 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp @@ -0,0 +1,82 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_THL_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_THL_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_thl_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for thetal + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_thl_parameters( + const Spack& wthlsec, + const Spack& sqrtw2, + const Spack& sqrtthl, + const Spack& thlsec, + const Spack& thl_first, + const Spack& w1_1, + const Spack& w1_2, + const Spack& Skew_w, + const Spack& a, + const Scalar thl_tol, + const Scalar w_thresh, + Spack& thl1_1, + Spack& thl1_2, + Spack& thl2_1, + Spack& thl2_2, + Spack& sqrtthl2_1, + Spack& sqrtthl2_2) +{ + thl1_1 = thl_first; + thl1_2 = thl_first; + thl2_1 = 0; + thl2_2 = 0; + sqrtthl2_1 = 0; + sqrtthl2_2 = 0; + + const Smask condition = thlsec > (thl_tol*thl_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; + + const Spack corrtest1 = ekat::max(-1, ekat::min(1, wthlsec/(sqrtw2*sqrtthl))); + const Spack tmp_val_1(-corrtest1/w1_1), tmp_val_2(-corrtest1/w1_2); + + Spack Skew_thl(0); + if (SC::dothetal_skew == true) { + const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); + Skew_thl.set(tsign>sp(0.4), sp(1.2)*Skew_w); + Skew_thl.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); + } + + if (condition.any()) { + thl2_1.set(condition, + ekat::min(100, + ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1-a)*ekat::square(tmp_val_1)) + - (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*a*(tmp_val_1 - tmp_val_2))))*thlsec); + thl2_2.set(condition, + ekat::min(100, + ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) + - (1 - a)*ekat::square(tmp_val_1)) + + (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*thlsec); + + thl1_1.set(condition, tmp_val_2*sqrtthl+thl_first); + thl1_2.set(condition, tmp_val_1*sqrtthl+thl_first); + + sqrtthl2_1.set(condition, ekat::sqrt(thl2_1)); + sqrtthl2_2.set(condition, ekat::sqrt(thl2_2)); + } +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp new file mode 100644 index 000000000000..2eaee89a72fc --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp @@ -0,0 +1,32 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_TILDE_TO_REAL_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_TILDE_TO_REAL_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_tilde_to_real. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Convert tilde variables to "real" variables + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_tilde_to_real( + const Spack& w_first, + const Spack& sqrtw2, + Spack& w1) +{ + w1 *= sqrtw2; + w1 += w_first; +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp new file mode 100644 index 000000000000..601622ee0f45 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp @@ -0,0 +1,61 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_VV_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_VV_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc shoc_assumed_pdf_vv_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for vertical velocity + */ + +template +KOKKOS_FUNCTION +void Functions::shoc_assumed_pdf_vv_parameters( + const Spack& w_first, + const Spack& w_sec, + const Spack& w3var, + const Scalar w_tol_sqd, + Spack& Skew_w, + Spack& w1_1, + Spack& w1_2, + Spack& w2_1, + Spack& w2_2, + Spack& a) +{ + Skew_w = 0; + w1_1 = w_first; + w1_2 = w_first; + w2_1 = 0; + w2_2 = 0; + a = 0.5; + + const Smask condition = w_sec > w_tol_sqd; + + const Scalar tmp_val(0.4); + const Scalar one_m_tmp_val(1 - tmp_val); + const Scalar sqrtw2t(std::sqrt(1-tmp_val)); + + Skew_w.set(condition, w3var/ekat::sqrt(ekat::cube(w_sec))); + a.set(condition, + ekat::max(sp(0.01), + ekat::min(sp(0.99), + sp(0.5)*(1 - Skew_w*ekat::sqrt(1/(4*(one_m_tmp_val*one_m_tmp_val*one_m_tmp_val) + + ekat::square(Skew_w))))))); + + w1_1.set(condition, ekat::sqrt((1 - a)/a)*sqrtw2t); + w1_2.set(condition, -1*ekat::sqrt(a/(1 - a))*sqrtw2t); + w2_1.set(condition, tmp_val*w_sec); + w2_2.set(condition, tmp_val*w_sec); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index afb9f618d52b..d3cd91920586 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -412,9 +412,9 @@ struct Functions #ifdef SCREAM_SMALL_KERNELS static void diag_second_shoc_moments_disp( const Int& shcol, const Int& nlev, const Int& nlevi, - const Scalar& thl2tune, - const Scalar& qw2tune, - const Scalar& qwthl2tune, + const Scalar& thl2tune, + const Scalar& qw2tune, + const Scalar& qwthl2tune, const Scalar& w2tune, const view_2d& thetal, const view_2d& qw, @@ -780,6 +780,154 @@ struct Functions const view_2d& shoc_ql2); #endif + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_buoyancy_flux( + const Spack& wthlsec, + const Spack& wqwsec, + const Spack& pval, + const Spack& wqls, + Spack& wthv_sec); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_cloud_liquid_variance( + const Spack& a, + const Spack& s1, + const Spack& ql1, + const Spack& C1, + const Spack& std_s1, + const Spack& s2, + const Spack& ql2, + const Spack& C2, + const Spack& std_s2, + const Spack& shoc_ql, + Spack& shoc_ql2); + + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_liquid_water_flux( + const Spack& a, + const Spack& w1_1, + const Spack& w_first, + const Spack& ql1, + const Spack& w1_2, + const Spack& ql2, + Spack& wqls); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_qs( + const Spack& Tl1_1, + const Spack& Tl1_2, + const Spack& pval, + const Smask& active_entries, + Spack& qs1, + Spack& beta1, + Spack& qs2, + Spack& beta2); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_s( + const Spack& qw1, + const Spack& qs, + const Spack& beta, + const Spack& pval, + const Spack& thl2, + const Spack& qw2, + const Spack& sqrtthl2, + const Spack& sqrtqw2, + const Spack& r_qwthl, + Spack& s, + Spack& std_s, + Spack& qn, + Spack& C); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_sgs_liquid( + const Spack& a, + const Spack& ql1, + const Spack& ql2, + Spack& shoc_ql); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_compute_temperature( + const Spack& thl1, + const Spack& pval, + Spack& Tl1); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_inplume_correlations( + const Spack& sqrtqw2_1, + const Spack& sqrtthl2_1, + const Spack& a, + const Spack& sqrtqw2_2, + const Spack& sqrtthl2_2, + const Spack& qwthlsec, + const Spack& qw1_1, + const Spack& qw_first, + const Spack& thl1_1, + const Spack& thl_first, + const Spack& qw1_2, + const Spack& thl1_2, + Spack& r_qwthl_1); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_qw_parameters( + const Spack& wqwsec, + const Spack& sqrtw2, + const Spack& Skew_w, + const Spack& sqrtqt, + const Spack& qwsec, + const Spack& w1_2, + const Spack& w1_1, + const Spack& qw_first, + const Spack& a, + const Scalar rt_tol, + const Scalar w_thresh, + Spack& qw1_1, + Spack& qw1_2, + Spack& qw2_1, + Spack& qw2_2, + Spack& sqrtqw2_1, + Spack& sqrtqw2_2); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_thl_parameters( + const Spack& wthlsec, + const Spack& sqrtw2, + const Spack& sqrtthl, + const Spack& thlsec, + const Spack& thl_first, + const Spack& w1_1, + const Spack& w1_2, + const Spack& Skew_w, + const Spack& a, + const Scalar thl_tol, + const Scalar w_thresh, + Spack& thl1_1, + Spack& thl1_2, + Spack& thl2_1, + Spack& thl2_2, + Spack& sqrtthl2_1, + Spack& sqrtthl2_2); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_tilde_to_real( + const Spack& w_first, + const Spack& sqrtw2, + Spack& w1); + + KOKKOS_FUNCTION + static void shoc_assumed_pdf_vv_parameters( + const Spack& w_first, + const Spack& w_sec, + const Spack& w3var, + const Scalar w_tol_sqd, + Spack& Skew_w, + Spack& w1_1, + Spack& w1_2, + Spack& w2_1, + Spack& w2_2, + Spack& a); + KOKKOS_FUNCTION static void compute_shr_prod( const MemberType& team, @@ -1246,6 +1394,16 @@ struct Functions # include "shoc_update_prognostics_implicit_impl.hpp" # include "shoc_diag_third_shoc_moments_impl.hpp" # include "shoc_assumed_pdf_impl.hpp" +# include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" +# include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" +# include "shoc_assumed_pdf_compute_qs_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" +# include "shoc_assumed_pdf_compute_temperature_impl.hpp" +# include "shoc_assumed_pdf_qw_parameters_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_thl_parameters_impl.hpp" +# include "shoc_assumed_pdf_vv_parameters_impl.hpp" # include "shoc_adv_sgs_tke_impl.hpp" # include "shoc_compute_tmpi_impl.hpp" # include "shoc_integ_column_stability_impl.hpp" From 0a92aaaa9ca2333124a9b0b6452665b68a979f94 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 1 Aug 2024 16:01:07 -0600 Subject: [PATCH 275/477] GPU fixes: missing includes, and accessing constants on dev --- .../shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp | 5 ++++- components/eamxx/src/physics/shoc/shoc_functions.hpp | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp index e857951c5f2b..eb5f80c2d8f2 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp @@ -20,7 +20,10 @@ void Functions::shoc_assumed_pdf_compute_temperature( const Spack& pval, Spack& Tl1) { - Tl1 = thl1/(ekat::pow(C::P0/pval,(C::Rair/C::CP))); + constexpr Scalar basepres = C::P0; + constexpr Scalar rair = C::Rair; + constexpr Scalar cp = C::CP; + Tl1 = thl1/(ekat::pow(basepres/pval,(rair/cp))); } } // namespace shoc diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index d3cd91920586..08a0d4976feb 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -1394,15 +1394,18 @@ struct Functions # include "shoc_update_prognostics_implicit_impl.hpp" # include "shoc_diag_third_shoc_moments_impl.hpp" # include "shoc_assumed_pdf_impl.hpp" +# include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" # include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" # include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" # include "shoc_assumed_pdf_compute_qs_impl.hpp" # include "shoc_assumed_pdf_compute_s_impl.hpp" # include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" # include "shoc_assumed_pdf_compute_temperature_impl.hpp" +# include "shoc_assumed_pdf_inplume_correlations_impl.hpp" # include "shoc_assumed_pdf_qw_parameters_impl.hpp" # include "shoc_assumed_pdf_compute_s_impl.hpp" # include "shoc_assumed_pdf_thl_parameters_impl.hpp" +# include "shoc_assumed_pdf_tilde_to_real_impl.hpp" # include "shoc_assumed_pdf_vv_parameters_impl.hpp" # include "shoc_adv_sgs_tke_impl.hpp" # include "shoc_compute_tmpi_impl.hpp" From 26fab5c3448eb1864122add7572904794b139f88 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 1 Aug 2024 16:07:58 -0600 Subject: [PATCH 276/477] Use KOKKOS_INLINE_FUNCTION --- .../eamxx/src/physics/shoc/CMakeLists.txt | 12 ----- ...shoc_assumed_pdf_compute_buoyancy_flux.cpp | 14 ----- ...umed_pdf_compute_cloud_liquid_variance.cpp | 14 ----- ..._assumed_pdf_compute_liquid_water_flux.cpp | 14 ----- .../shoc/eti/shoc_assumed_pdf_compute_qs.cpp | 14 ----- .../shoc/eti/shoc_assumed_pdf_compute_s.cpp | 14 ----- .../shoc_assumed_pdf_compute_sgs_liquid.cpp | 14 ----- .../shoc_assumed_pdf_compute_temperature.cpp | 14 ----- .../shoc_assumed_pdf_inplume_correlations.cpp | 14 ----- .../eti/shoc_assumed_pdf_qw_parameters.cpp | 14 ----- .../eti/shoc_assumed_pdf_thl_parameters.cpp | 14 ----- .../eti/shoc_assumed_pdf_tilde_to_real.cpp | 14 ----- .../eti/shoc_assumed_pdf_vv_parameters.cpp | 14 ----- ...assumed_pdf_compute_buoyancy_flux_impl.hpp | 2 +- ...pdf_compute_cloud_liquid_variance_impl.hpp | 2 +- ...med_pdf_compute_liquid_water_flux_impl.hpp | 2 +- .../impl/shoc_assumed_pdf_compute_qs_impl.hpp | 2 +- .../impl/shoc_assumed_pdf_compute_s_impl.hpp | 2 +- ...oc_assumed_pdf_compute_sgs_liquid_impl.hpp | 2 +- ...c_assumed_pdf_compute_temperature_impl.hpp | 2 +- ..._assumed_pdf_inplume_correlations_impl.hpp | 2 +- .../shoc_assumed_pdf_qw_parameters_impl.hpp | 2 +- .../shoc_assumed_pdf_thl_parameters_impl.hpp | 2 +- .../shoc_assumed_pdf_tilde_to_real_impl.hpp | 2 +- .../shoc_assumed_pdf_vv_parameters_impl.hpp | 2 +- .../eamxx/src/physics/shoc/shoc_functions.hpp | 52 ++++++++++--------- 26 files changed, 39 insertions(+), 217 deletions(-) delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp delete mode 100644 components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index e6724622a928..891626b9d3df 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -25,18 +25,6 @@ if (NOT EAMXX_ENABLE_GPU) list(APPEND SHOC_SRCS eti/shoc_adv_sgs_tke.cpp eti/shoc_assumed_pdf.cpp - eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp - eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp - eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp - eti/shoc_assumed_pdf_compute_qs.cpp - eti/shoc_assumed_pdf_compute_sgs_liquid.cpp - eti/shoc_assumed_pdf_compute_temperature.cpp - eti/shoc_assumed_pdf_inplume_correlations.cpp - eti/shoc_assumed_pdf_qw_parameters.cpp - eti/shoc_assumed_pdf_compute_s.cpp - eti/shoc_assumed_pdf_thl_parameters.cpp - eti/shoc_assumed_pdf_tilde_to_real.cpp - eti/shoc_assumed_pdf_vv_parameters.cpp eti/shoc_calc_shoc_varorcovar.cpp eti/shoc_calc_shoc_vertflux.cpp eti/shoc_check_length_scale_shoc_length.cpp diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp deleted file mode 100644 index 4d38b8525dd8..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_buoyancy_flux.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_buoyancy_flux on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp deleted file mode 100644 index eb83e2098772..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_cloud_liquid_variance.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_cloud_liquid_variance on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp deleted file mode 100644 index d8f99806cf9b..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_liquid_water_flux.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_liquid_water_flux on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp deleted file mode 100644 index fbac9733184b..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_qs.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_qs_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_qs on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp deleted file mode 100644 index b74ae269a764..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_s.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_s_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_s on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp deleted file mode 100644 index d2c786e88368..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_sgs_liquid.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_sgs_liquid on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp deleted file mode 100644 index 8fdf36d64088..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_compute_temperature.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_compute_temperature_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_compute_temperature on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp deleted file mode 100644 index d29641dd73bf..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_inplume_correlations.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_inplume_correlations_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_inplume_correlations on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp deleted file mode 100644 index b9a9501b9887..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_qw_parameters.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_qw_parameters_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_qw_parameters on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp deleted file mode 100644 index 716fd7cd3d29..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_thl_parameters.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_thl_parameters_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_thl_parameters on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp deleted file mode 100644 index 250e3f1a0a40..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_tilde_to_real.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_tilde_to_real_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_tilde_to_real on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp b/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp deleted file mode 100644 index 2af26cfc1adb..000000000000 --- a/components/eamxx/src/physics/shoc/eti/shoc_assumed_pdf_vv_parameters.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "shoc_assumed_pdf_vv_parameters_impl.hpp" - -namespace scream { -namespace shoc { - -/* - * Explicit instantiation for doing shoc_assumed_pdf_vv_parameters on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp index db31f1ab1bd5..7a0633fe7f34 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_buoyancy_flux( const Spack& wthlsec, const Spack& wqwsec, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp index 06f740d6eb55..dac01954db31 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_cloud_liquid_variance( const Spack& a, const Spack& s1, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp index b225494e7b6d..ceb64c533a37 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_liquid_water_flux( const Spack& a, const Spack& w1_1, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp index 3949f087bfe2..867ffe3fed3a 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp @@ -15,7 +15,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_qs( const Spack& Tl1_1, const Spack& Tl1_2, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp index 180bf889d362..70fb386055a1 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_s( const Spack& qw1, const Spack& qs, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp index cbc12f99a673..837fe90e2991 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_sgs_liquid( const Spack& a, const Spack& ql1, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp index eb5f80c2d8f2..e60a744ae93f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp @@ -14,7 +14,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_compute_temperature( const Spack& thl1, const Spack& pval, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp index 52554ea9c98a..600af5a6f51b 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_inplume_correlations( const Spack& sqrtqw2_1, const Spack& sqrtthl2_1, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp index 546be97f6fb4..d4cb020a7fa2 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_qw_parameters( const Spack& wqwsec, const Spack& sqrtw2, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp index 90d60e439b56..686786d6fe21 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_thl_parameters( const Spack& wthlsec, const Spack& sqrtw2, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp index 2eaee89a72fc..e2de2705fb8e 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_tilde_to_real( const Spack& w_first, const Spack& sqrtw2, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp index 601622ee0f45..6cd237567597 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp @@ -16,7 +16,7 @@ namespace shoc { */ template -KOKKOS_FUNCTION +KOKKOS_INLINE_FUNCTION void Functions::shoc_assumed_pdf_vv_parameters( const Spack& w_first, const Spack& w_sec, diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 08a0d4976feb..d1d547245b1f 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -780,7 +780,7 @@ struct Functions const view_2d& shoc_ql2); #endif - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_buoyancy_flux( const Spack& wthlsec, const Spack& wqwsec, @@ -788,7 +788,7 @@ struct Functions const Spack& wqls, Spack& wthv_sec); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_cloud_liquid_variance( const Spack& a, const Spack& s1, @@ -803,7 +803,7 @@ struct Functions Spack& shoc_ql2); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_liquid_water_flux( const Spack& a, const Spack& w1_1, @@ -813,7 +813,7 @@ struct Functions const Spack& ql2, Spack& wqls); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_qs( const Spack& Tl1_1, const Spack& Tl1_2, @@ -824,7 +824,7 @@ struct Functions Spack& qs2, Spack& beta2); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_s( const Spack& qw1, const Spack& qs, @@ -840,20 +840,20 @@ struct Functions Spack& qn, Spack& C); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_sgs_liquid( const Spack& a, const Spack& ql1, const Spack& ql2, Spack& shoc_ql); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_compute_temperature( const Spack& thl1, const Spack& pval, Spack& Tl1); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_inplume_correlations( const Spack& sqrtqw2_1, const Spack& sqrtthl2_1, @@ -869,7 +869,7 @@ struct Functions const Spack& thl1_2, Spack& r_qwthl_1); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_qw_parameters( const Spack& wqwsec, const Spack& sqrtw2, @@ -889,7 +889,7 @@ struct Functions Spack& sqrtqw2_1, Spack& sqrtqw2_2); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_thl_parameters( const Spack& wthlsec, const Spack& sqrtw2, @@ -909,13 +909,13 @@ struct Functions Spack& sqrtthl2_1, Spack& sqrtthl2_2); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_tilde_to_real( const Spack& w_first, const Spack& sqrtw2, Spack& w1); - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION static void shoc_assumed_pdf_vv_parameters( const Spack& w_first, const Spack& w_sec, @@ -1394,19 +1394,6 @@ struct Functions # include "shoc_update_prognostics_implicit_impl.hpp" # include "shoc_diag_third_shoc_moments_impl.hpp" # include "shoc_assumed_pdf_impl.hpp" -# include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" -# include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" -# include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" -# include "shoc_assumed_pdf_compute_qs_impl.hpp" -# include "shoc_assumed_pdf_compute_s_impl.hpp" -# include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" -# include "shoc_assumed_pdf_compute_temperature_impl.hpp" -# include "shoc_assumed_pdf_inplume_correlations_impl.hpp" -# include "shoc_assumed_pdf_qw_parameters_impl.hpp" -# include "shoc_assumed_pdf_compute_s_impl.hpp" -# include "shoc_assumed_pdf_thl_parameters_impl.hpp" -# include "shoc_assumed_pdf_tilde_to_real_impl.hpp" -# include "shoc_assumed_pdf_vv_parameters_impl.hpp" # include "shoc_adv_sgs_tke_impl.hpp" # include "shoc_compute_tmpi_impl.hpp" # include "shoc_integ_column_stability_impl.hpp" @@ -1425,4 +1412,19 @@ struct Functions #endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE +// Some functions should be inlined, thus do not use ETI +# include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" +# include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" +# include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" +# include "shoc_assumed_pdf_compute_qs_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" +# include "shoc_assumed_pdf_compute_temperature_impl.hpp" +# include "shoc_assumed_pdf_inplume_correlations_impl.hpp" +# include "shoc_assumed_pdf_qw_parameters_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_thl_parameters_impl.hpp" +# include "shoc_assumed_pdf_tilde_to_real_impl.hpp" +# include "shoc_assumed_pdf_vv_parameters_impl.hpp" + #endif // SHOC_FUNCTIONS_HPP From cd19cdfe58780516c55f7776ff9f4c61f379a719 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 7 Aug 2024 10:10:31 -0600 Subject: [PATCH 277/477] EAMxx: fix access to output file in IO Avoid calling scorpio interfaces on a file not open --- components/eamxx/src/share/io/scream_output_manager.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index d6be54d09181..0b738efe6483 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -393,7 +393,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) snapshot_start = m_case_t0; snapshot_start += m_time_bnds[0]; } - if (not filespecs.storage.snapshot_fits(snapshot_start)) { + if (filespecs.is_open and not filespecs.storage.snapshot_fits(snapshot_start)) { release_file(filespecs.filename); filespecs.close(); } @@ -496,7 +496,8 @@ void OutputManager::run(const util::TimeStamp& timestamp) scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_filename",m_output_file_specs.filename); scorpio::set_attribute (filespecs.filename,"GLOBAL","num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); - int nsnaps = scorpio::get_dimlen(m_output_file_specs.filename,"time"); + int nsnaps = m_output_file_specs.is_open + ? scorpio::get_dimlen(m_output_file_specs.filename,"time") : 0; scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_file_num_snaps",nsnaps); } // Write these in both output and rhist file. The former, b/c we need these info when we postprocess From 2d208212c0bedab148ec90052ef4c06d730d25fb Mon Sep 17 00:00:00 2001 From: dqwu Date: Fri, 9 Aug 2024 11:25:21 -0500 Subject: [PATCH 278/477] Disable hipInit call before MPI_Init on Frontier The hipInit workaround was originally introduced to prevent occasional segmentation faults during MPI_Init. With the cpe/22.12 configuration in the post-maintenance setup on Frontier, this workaround is no longer necessary. Additionally, it seems that the hipInit call may be causing non-deterministic DIRK NaN errors in recent ne1024 DECADAL runs. So far, no DIRK errors have been observed when hipInit is not invoked. --- .../cmake_macros/crayclang-scream_frontier-scream-gpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index a5c89c3318ed..d42597be2e0b 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -5,7 +5,7 @@ set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") -string(APPEND CPPDEFS " -DLINUX -DSCREAM_SYSTEM_WORKAROUND=1") +string(APPEND CPPDEFS " -DLINUX -DSCREAM_SYSTEM_WORKAROUND=0") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() From c35785da4eae80c0bae19aa1266888b961367a2b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 12 Aug 2024 23:07:55 -0700 Subject: [PATCH 279/477] Fixes an array index in scream coupler F90 file --- components/eamxx/src/mct_coupling/scream_cpl_indices.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index e754d2fef1b0..dc5be4de373f 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -111,8 +111,8 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(15) = mct_avect_indexra(x2a,'Sf_ofrac') import_cpl_indices(16) = mct_avect_indexra(x2a,'Sf_lfrac') import_cpl_indices(17) = mct_avect_indexra(x2a,'Sf_ifrac') - import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_fv') - import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_ram1') + import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_fv') + import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_ram1') ! Vector components import_vector_components(11) = 0 From 0ae7719dce253f8bf057be0b463487cc64062536 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 21 Feb 2024 14:31:14 -0800 Subject: [PATCH 280/477] Adds surface and online emission interface and a test --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- ...and_online_emissions_process_interface.cpp | 159 ++++++++++++++++++ ...and_online_emissions_process_interface.hpp | 62 +++++++ .../eamxx/src/physics/register_physics.hpp | 2 + 4 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index c5feb52e12a0..9138bba6d5a2 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -45,7 +45,8 @@ add_library(mam eamxx_mam_optics_process_interface.cpp eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp - eamxx_mam_wetscav_process_interface.cpp) + eamxx_mam_wetscav_process_interface.cpp + eamxx_mam_srf_and_online_emissions_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp new file mode 100644 index 000000000000..c107dbe6970a --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -0,0 +1,159 @@ +#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" + +/* +Future work: +Wirte comments +write in/outs for all variables clearly +*/ + +namespace scream { + +// ========================================================================================= +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::set_grids( + const std::shared_ptr grids_manager) { + + using namespace ekat::units; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] + /*add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa] + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + add_field("obklen", scalar2d_layout, m, grid_name); + add_field("surfric", scalar2d_layout, m / s, grid_name); + + auto nondim = ekat::units::Units::nondimensional(); + add_field("landfrac", scalar2d_layout, nondim, grid_name); + //add_field("icefrac", scalar2d_layout, nondim, grid_name); + //add_field("ocnfrac", scalar2d_layout, nondim, grid_name); + add_field("fv", scalar2d_layout, m / s, grid_name); + add_field("ram1", scalar2d_layout, s / m, grid_name); + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + // printf("%s \n", int_nmr_field_name); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + }*/ +} + +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + // Gather runtime options + //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::run_impl(const double dt) { + std::cout << "End of derydep run" << std::endl; +} + +// ========================================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp new file mode 100644 index 000000000000..1ad5e4e6da68 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -0,0 +1,62 @@ +#ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#define EAMXX_MAM_SRF_ONLINE_EMISS_HPP + +// For declaring surface and online emission class derived from atm process class +#include + +// For MAM4 aerosol configuration +#include + +// For component name +#include + +namespace scream { + +// The process responsible for handling MAM4 surface and online emissions. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + public: + // Constructor + MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_srf_online_emissions"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMSrfOnlineEmiss + +} // namespace scream + +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index aea9f2f543a9..f8d3a9edb199 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -29,6 +29,7 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" #include "physics/mam/eamxx_mam_aci_process_interface.hpp" #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" +#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -68,6 +69,7 @@ inline void register_physics () { proc_factory.register_product("mam4_drydep",&create_atmosphere_process); proc_factory.register_product("mam4_aci",&create_atmosphere_process); proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); + proc_factory.register_product("mam4_srf_online_emiss",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 0223b78cd7222ee45e9c9392a2c2b8fb4a01cd0b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 21 Feb 2024 17:36:04 -0800 Subject: [PATCH 281/477] Adds some inputs and preprocess struct, test works now --- ...and_online_emissions_process_interface.cpp | 96 +++++++++++++++---- ...and_online_emissions_process_interface.hpp | 60 +++++++++++- 2 files changed, 133 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index c107dbe6970a..f7396950be3c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -9,7 +9,8 @@ write in/outs for all variables clearly namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -19,7 +20,6 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::Paramet // ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { - using namespace ekat::units; // The units of mixing ratio Q are technically non-dimensional. @@ -42,16 +42,17 @@ void MAMSrfOnlineEmiss::set_grids( // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - - // Layout for 2D (2d horiz) variable + const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + + // Layout for 2D (2d horiz) variable const FieldLayout scalar2d_layout{{COL}, {ncol_}}; - + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] - /*add_field("p_mid", scalar3d_layout_mid, Pa, + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, grid_name); // total pressure @@ -67,17 +68,9 @@ void MAMSrfOnlineEmiss::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); - add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); - add_field("obklen", scalar2d_layout, m, grid_name); - add_field("surfric", scalar2d_layout, m / s, grid_name); - - auto nondim = ekat::units::Units::nondimensional(); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - //add_field("icefrac", scalar2d_layout, nondim, grid_name); - //add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - add_field("fv", scalar2d_layout, m / s, grid_name); - add_field("ram1", scalar2d_layout, s / m, grid_name); + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -119,7 +112,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - }*/ + } } // ========================================================================================= @@ -140,14 +133,77 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); - EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), - "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // set up our preprocess functor + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1ad5e4e6da68..d31cd9075c6a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,7 +1,8 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP -// For declaring surface and online emission class derived from atm process class +// For declaring surface and online emission class derived from atm process +// class #include // For MAM4 aerosol configuration @@ -12,12 +13,21 @@ namespace scream { -// The process responsible for handling MAM4 surface and online emissions. The AD -// stores exactly ONE instance of this class in its list of subcomponents. +// The process responsible for handling MAM4 surface and online emissions. The +// AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { + using KT = ekat::KokkosTypes; + // number of horizontal columns and vertical levels int ncol_, nlev_; + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + // buffer for sotring temporary variables mam_coupling::Buffer buffer_; @@ -54,6 +64,50 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); + team.team_barrier(); + + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMAci::Preprocess + + private: + // preprocessing scratch pad + Preprocess preprocess_; }; // MAMSrfOnlineEmiss From 30f0494b2348f24790f0bd81d19fb3af94c07edb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 22 Feb 2024 11:33:04 -0800 Subject: [PATCH 282/477] Adds some notes for the integration --- ...and_online_emissions_process_interface.cpp | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f7396950be3c..f3d2a5a04ba6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -208,7 +208,50 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - std::cout << "End of derydep run" << std::endl; + + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + /* Rough notes: + + Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two + subroutines, aero_model_emissions and set_srf_emissions. + + Here is the code: + + ! initialize chemistry constituent surface fluxes to zero + do m = 2,pcnst + n = map2chm(m) + if (n>0) cam_in%cflx(:,m) = 0._r8 + enddo + + ! aerosol emissions ... + call aero_model_emissions( state, & ! in + cam_in ) ! out + + ! prescribed emissions from file ... + + !----------------------------------------------------------------------- + ! ... Set surface emissions + !----------------------------------------------------------------------- + call set_srf_emissions( lchnk, ncol, sflx(:,:) ) + + do m = 1,pcnst + n = map2chm(m) + if ( n /= h2o_ndx .and. n > 0 ) then + cam_in%cflx(:ncol,m) = cam_in%cflx(:ncol,m) + sflx(:ncol,n) + call outfld( sflxnam(m), cam_in%cflx(:ncol,m), ncol,lchnk ) + endif + enddo + + + */ + + std::cout << "End of surface emissions run" << std::endl; } // ========================================================================================= From 2bdad85e381ecc956519372e013b43b12b2c8888 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Jul 2024 06:38:07 -0700 Subject: [PATCH 283/477] A working test in new single-process test folder --- ...and_online_emissions_process_interface.cpp | 196 ++++++++++-------- .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../mam/emissions/CMakeLists.txt | 44 ++++ .../single-process/mam/emissions/input.yaml | 37 ++++ .../single-process/mam/emissions/output.yaml | 61 ++++++ 5 files changed, 253 insertions(+), 86 deletions(-) create mode 100644 components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/emissions/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/emissions/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f3d2a5a04ba6..e41e41dc73ef 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -17,20 +17,18 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - q_unit.set_string("kg/kg"); - - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); + // set grid for all the inputs and outputs + // use physics grid + grid_ = grids_manager->get_grid("Physics"); - grid_ = grids_manager->get_grid("Physics"); + // Name of the grid const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -39,93 +37,114 @@ void MAMSrfOnlineEmiss::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // Layout for 3D (2d horiz X 1d vertical) variables + // mid points + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + // layout for 2D (1d horiz X 1d vertical) variable + FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + auto nondim = ekat::units::Units::nondimensional(); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + // atmospheric quantities + // specific humidity [kg/kg] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_layout_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_layout_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + + // planetary boundary layer height + add_field("pbl_height", scalar2d_layout_col, m, grid_name); + + // ======================================================================== + // Output from this whole process + // ======================================================================== + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - + mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - } - } - // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - // printf("%s \n", int_nmr_field_name); - - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - + mam_coupling::cld_aero_mmr_field_name(mode, a); if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } - } - } + } // end for loop num species + } // end for loop for num modes - // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } -} + } // end for loop num gases -// ========================================================================================= -// ON HOST, returns the number of bytes of device memory needed by the above -// Buffer type given the number of columns and vertical levels +} // function set_grids ends + +// ================================================================ +// INIT_BUFFERS +// ================================================================ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -// ========================================================================================= -// ON HOST, initializeŃ• the Buffer type with sufficient memory to store -// intermediate (dry) quantities on the given number of columns with the given -// number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -138,26 +157,31 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ + void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -208,7 +232,6 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -218,15 +241,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two - subroutines, aero_model_emissions and set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in + chemistry.F90. Basically call two subroutines, aero_model_emissions and + set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -235,9 +259,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -248,7 +272,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index 95e4d81ccbbf..62435db229fe 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -23,6 +23,7 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam/aci) add_subdirectory(mam/drydep) add_subdirectory(mam/wet_scav) + add_subdirectory(mam/emissions) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt new file mode 100644 index 000000000000..b57adcb60ef0 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -0,0 +1,44 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_srf_online_emiss_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_srf_online_emiss physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_srf_online_emiss physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml new file mode 100644 index 000000000000..741cf2970320 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -0,0 +1,37 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_srf_online_emiss] + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + pbl_height : 1.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml new file mode 100644 index 000000000000..7363b899a5fa --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -0,0 +1,61 @@ +%YAML 1.1 +--- +filename_prefix: mam4_srf_online_emiss_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 +output_control: + Frequency: 1 + frequency_units: nsteps +... From 9948fa85ae6e7664fc2ea6fd96f08d82bd717fde Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Jul 2024 11:19:45 -0700 Subject: [PATCH 284/477] Adds horiz wind vector and some cleanup; test is working --- ...and_online_emissions_process_interface.cpp | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index e41e41dc73ef..fe415e9436ca 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -39,12 +39,14 @@ void MAMSrfOnlineEmiss::set_grids( // Layout for 3D (2d horiz X 1d vertical) variables // mid points - FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; // interfaces - FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // vector layout + FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true,2); // layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + FieldLayout scalar2d_col{{COL}, {ncol_}}; using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers @@ -57,37 +59,40 @@ void MAMSrfOnlineEmiss::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // atmospheric quantities // specific humidity [kg/kg] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud liquid number mixing ratio [1/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); // cloud ice number mixing ratio [1/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); // Temperature[K] at midpoints - add_field("T_mid", scalar3d_layout_mid, K, grid_name); + add_field("T_mid", scalar3d_mid, K, grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + add_field("omega", scalar3d_mid, Pa / s, grid_name); // Total pressure [Pa] at midpoints - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + add_field("p_mid", scalar3d_mid, Pa, grid_name); // Total pressure [Pa] at interfaces - add_field("p_int", scalar3d_layout_int, Pa, grid_name); + add_field("p_int", scalar3d_int, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - // planetary boundary layer height - add_field("pbl_height", scalar2d_layout_col, m, grid_name); + // Planetary boundary layer height + add_field("pbl_height", scalar2d_col, m, grid_name); + + // Horizontal winds (U and V wind components) [m/s] + add_field("horiz_winds", vector3d_mid, m/s, grid_name); // ======================================================================== // Output from this whole process @@ -99,7 +104,7 @@ void MAMSrfOnlineEmiss::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios @@ -107,7 +112,7 @@ void MAMSrfOnlineEmiss::set_grids( // NOT advected const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { @@ -115,7 +120,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -124,7 +129,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } // end for loop num species @@ -132,7 +137,7 @@ void MAMSrfOnlineEmiss::set_grids( for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // end for loop num gases From c9f8c52fef3d0c439cabcec980b1c7a42db5121a Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 15 Apr 2024 12:23:13 -0600 Subject: [PATCH 285/477] get scream in correct place for emissions --- ...and_online_emissions_process_interface.cpp | 209 ++++++++---------- 1 file changed, 90 insertions(+), 119 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index fe415e9436ca..f3d2a5a04ba6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -17,18 +17,20 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ================================================================ -// SET_GRIDS -// ================================================================ +// ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - // set grid for all the inputs and outputs - // use physics grid - grid_ = grids_manager->get_grid("Physics"); + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); - // Name of the grid + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -37,119 +39,93 @@ void MAMSrfOnlineEmiss::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variables - // mid points - FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // vector layout - FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true,2); + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_col{{COL}, {ncol_}}; - - using namespace ekat::units; - auto q_unit = kg / kg; // units of mass mixing ratios of tracers - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - - auto nondim = ekat::units::Units::nondimensional(); + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - // atmospheric quantities - // specific humidity [kg/kg] - add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud liquid mass mixing ratio [kg/kg] - add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud ice mass mixing ratio [kg/kg] - add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud liquid number mixing ratio [1/kg] - add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); - - // cloud ice number mixing ratio [1/kg] - add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); - - // Temperature[K] at midpoints - add_field("T_mid", scalar3d_mid, K, grid_name); - - // Vertical pressure velocity [Pa/s] at midpoints - add_field("omega", scalar3d_mid, Pa / s, grid_name); - - // Total pressure [Pa] at midpoints - add_field("p_mid", scalar3d_mid, Pa, grid_name); - - // Total pressure [Pa] at interfaces - add_field("p_int", scalar3d_int, Pa, grid_name); - - // Layer thickness(pdel) [Pa] at midpoints - add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - - // Planetary boundary layer height - add_field("pbl_height", scalar2d_col, m, grid_name); - - // Horizontal winds (U and V wind components) [m/s] - add_field("horiz_winds", vector3d_mid, m/s, grid_name); - - // ======================================================================== - // Output from this whole process - // ======================================================================== + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa] + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios - for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { - // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = - mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_mid, n_unit, + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - - // cloudborne aerosol tracers of interest: number (n) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected - const char *cld_nmr_field_name = - mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(mode, a); + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit, + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + // printf("%s \n", int_nmr_field_name); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(mode, a); + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_mid, q_unit, + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } - } // end for loop num species - } // end for loop for num modes + } + } + // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } // end for loop num gases - -} // function set_grids ends + } +} -// ================================================================ -// INIT_BUFFERS -// ================================================================ +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -162,31 +138,26 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ================================================================ -// INITIALIZE_IMPL -// ================================================================ - +// ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); - - // store fields converted to dry mmr from wet mmr in dry_atm_ - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -237,6 +208,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -246,16 +218,15 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions and - set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two + subroutines, aero_model_emissions and set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -264,9 +235,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -277,7 +248,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; From 407c5f365c7d41952037684f9141e8dd054843f8 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 8 Jul 2024 13:38:37 -0600 Subject: [PATCH 286/477] compiling and running test --- .../cime_config/namelist_defaults_scream.xml | 24 ++ ...and_online_emissions_process_interface.cpp | 354 ++++++++++++++---- ...and_online_emissions_process_interface.hpp | 15 +- .../src/physics/mam/mam_emissions_utils.hpp | 59 +++ 4 files changed, 366 insertions(+), 86 deletions(-) create mode 100644 components/eamxx/src/physics/mam/mam_emissions_utils.hpp diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index f09d10b0b320..bb36bdc2ec7f 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,6 +269,30 @@ be lost if SCREAM_HACK_XML is not enabled. + + + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f3d2a5a04ba6..b34186d7de82 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,16 +1,23 @@ -#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" +#include +#include + +#include "share/grid/point_grid.hpp" +#include "share/io/scorpio_input.hpp" + +// for SCREAM_CIME_BUILD +#include "scream_config.h" /* Future work: -Wirte comments +Write comments write in/outs for all variables clearly */ namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, - const ekat::ParameterList ¶ms) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm& comm, + const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -24,17 +31,15 @@ void MAMSrfOnlineEmiss::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - q_unit.set_string("kg/kg"); + Units q_unit(kg / kg, "kg/kg"); - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); + Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers - grid_ = grids_manager->get_grid("Physics"); - const auto &grid_name = grid_->name(); + grid_ = grids_manager->get_grid("Physics"); + const auto& grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -51,56 +56,56 @@ void MAMSrfOnlineEmiss::set_grids( // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] + grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] + grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure + grid_name); // total pressure add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] + grid_name); // pseudo density in [Pa] add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity + "tracers"); // specific humidity add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet + "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet + "tracers"); // ice cloud water [kg/kg] wet add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio + "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio + "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints + grid_name); // Vertical pressure velocity [Pa/s] at midpoints // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if (strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); // printf("%s \n", int_nmr_field_name); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if (strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } @@ -108,8 +113,8 @@ void MAMSrfOnlineEmiss::set_grids( } // aerosol-related gases: mass mixing ratios - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } @@ -123,13 +128,13 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { } // ========================================================================================= -// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. -void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG( - buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= + requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); @@ -137,76 +142,264 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } +// ========================================================================================= +// inline void set_emissions_layouts( +// const std::map map_spec_id, +// const std::string emis_type, +// std::map& host_views, +// mam_coupling::complex_view_2d::HostMirror& +// specrefndxsw_host, // complex refractive index for water visible +// mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { + +// // names take the form "online_emis_specifier_for_SO2" +// for (const auto& item : map_spec_id) { +// const auto spec_name = item.first; +// const int species_id = item.second; +// const auto file_name = emis_type + "_emis_specifier_" + spec_name; +// // const auto& fname = m_params.get(file_name); +// // update file name + + +// // read data +// AtmosphereInput srf_emissions_reader( +// params_srf_emissions, grid_, host_views_emissions, layouts_emissions); +// srf_emissions_reader.read_variables(); +// srf_emissions_reader.finalize(); +// } // end ispec + + + + // for (int i = 0; i < nswbands; i++) { + // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); + // specrefndxsw_host(i, species_id).imag() = + // haero::abs(host_views[sw_im_name](i)); + // } + // for (int i = 0; i < nlwbands; i++) { + // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); + // specrefndxlw_host(i, species_id).imag() = + // haero::abs(host_views[lw_im_name](i)); + // } + +// } // end // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // NOTE: these are taken as arguments to srf_emissions_inti() + // and then passed to trcdata_init() + // rmv_file = false; + // emis_cycle_yr + // emis_fixed_ymd + // emis_fixed_tod + // emis_type // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); + get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if (strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); + get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if (strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); + get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = - get_field_out(gas_mmr_field_name).get_view(); + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + // // read data from file + { + // // using namespace ShortFieldTagsNames; + + using view_1d_host = typename KT::view_1d::HostMirror; + + mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + + // { + // make a list of host views + std::map host_views_srf_emissions; + // defines layouts + std::map layouts_srf_emissions; + ekat::ParameterList params_srf_emissions; + std::string prefix_srf_emissions = "srf_emis_specifier_for_"; + + // // constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; + // // auto specrefndxsw_host = mam_coupling::complex_view_2d::HostMirror( + // // "specrefndxsw_host", nswbands_, maxd_aspectype); + + // // auto specrefndxlw_host = mam_coupling::complex_view_2d::HostMirror( + // // "specrefndxlw_host", nlwbands_, maxd_aspectype); + + std::map map_srf_emiss_name_species_id; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["num_a1"] = 3; + map_srf_emiss_name_species_id["num_a2"] = 4; + map_srf_emiss_name_species_id["num_a4"] = 5; + map_srf_emiss_name_species_id["pom_a4"] = 6; + map_srf_emiss_name_species_id["so4_a1"] = 7; + map_srf_emiss_name_species_id["so4_a2"] = 8; + + std::map map_online_emiss_name_species_id; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["num_a1"] = 3; + map_online_emiss_name_species_id["num_a2"] = 4; + map_online_emiss_name_species_id["num_a4"] = 5; + map_online_emiss_name_species_id["pom_a4"] = 6; + map_online_emiss_name_species_id["so4_a1"] = 7; + map_online_emiss_name_species_id["so4_a2"] = 8; + + // To create the input object, we need to set: + // 1) names (do during read loop) + // 2) params + // 3) host views + // 4) layouts + // set_emissions_names(surname_emissions, params_srf_emissions, + // host_views_emissions, layouts_emissions); + + // inline void set_emissions_names( + // const std::map map_spec_id, + // const std::string emis_type, + // std::map& host_views, + // mam_coupling::complex_view_2d::HostMirror& + // specrefndxsw_host, // complex refractive index for water visible + // mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { + + // // names take the form "online_emis_specifier_for_SO2" + // for (const auto& item : map_spec_id) { + // const auto spec_name = item.first; + // const int species_id = item.second; + // const auto file_name = emis_type + "_emis_specifier_" + spec_name; + // const auto& fname = m_params.get(file_name); + // update file name + + + // // read data + // AtmosphereInput srf_emissions_reader( + // params_srf_emissions, grid_, host_views_emissions, layouts_emissions); + // srf_emissions_reader.read_variables(); + // srf_emissions_reader.finalize(); + // } // end ispec + + + + // for (int i = 0; i < nswbands; i++) { + // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); + // specrefndxsw_host(i, species_id).imag() = + // haero::abs(host_views[sw_im_name](i)); + // } + // for (int i = 0; i < nlwbands; i++) { + // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); + // specrefndxlw_host(i, species_id).imag() = + // haero::abs(host_views[lw_im_name](i)); + // } + + // } // end + + // =============== + // Names + // =============== + + // names take the form _emis_specifier_for_ + // set_emissions_names("srf", params_srf_emissions, host_views_srf_emissions, layouts_srf_emissions); + // set_emissions_names("online", params_srf_emissions, host_views_online_emissions, layouts_online_emissions); + + // =============== + // Params + // =============== + using strvec_t = std::vector; + params_srf_emissions.set("Skip_Grid_Checks", true); + // params_srf_emissions.set("Field Names", {refindex_real_sw, refindex_im_sw, + // refindex_real_lw, refindex_im_lw}); + // params_srf_emissions.set("Filename", fname); + + // =============== + // Host Views + // =============== + using view_1d_host = typename KT::view_1d::HostMirror; + std::map host_views; + + // host_views[refindex_real_sw] = view_1d_host(refindex_real_sw, + // nswbands); host_views[refindex_im_sw] = view_1d_host(refindex_im_sw, + // nswbands); host_views[refindex_real_lw] = + // view_1d_host(refindex_real_lw, nlwbands); host_views[refindex_im_lw] = + // view_1d_host(refindex_im_lw, nlwbands); + + // =============== + // Layouts + // =============== + std::map layouts; + // FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; + // FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; + + // layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); + // layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); + // layouts.emplace(refindex_real_lw, scalar_refindex_lw_layout); + // layouts.emplace(refindex_im_lw, scalar_refindex_lw_layout); + + // reshape specrefndxsw_host and copy it to device + // mam4::modal_aer_opt::set_device_specrefindex( + // aerosol_optics_device_data_.specrefindex_sw, "short_wave", + // specrefndxsw_host); + // mam4::modal_aer_opt::set_device_specrefindex( + // aerosol_optics_device_data_.specrefindex_lw, "long_wave", + // specrefndxlw_host); + } } -// ========================================================================================= +// ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< @@ -218,15 +411,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two - subroutines, aero_model_emissions and set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in + chemistry.F90. Basically call two subroutines, aero_model_emissions and + set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -235,9 +429,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -248,11 +442,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; } -// ========================================================================================= -} // namespace scream +// ============================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index d31cd9075c6a..d59f0df56c1f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,14 +1,17 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#include +#include +#include +#include +// For MAM4 aerosol configuration +#include +#include // For declaring surface and online emission class derived from atm process // class #include - -// For MAM4 aerosol configuration -#include - -// For component name +// #include #include namespace scream { @@ -113,4 +116,4 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { } // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP \ No newline at end of file +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp new file mode 100644 index 000000000000..807896f12392 --- /dev/null +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -0,0 +1,59 @@ +#ifndef MAM_EMISSIONS_READ_TABLES_HPP +#define MAM_EMISSIONS_READ_TABLES_HPP + +#include "ekat/ekat_parameter_list.hpp" +#include "mam_coupling.hpp" +#include "share/field/field_manager.hpp" +#include "share/grid/abstract_grid.hpp" +#include "share/grid/grids_manager.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +// later to mam_coupling.hpp +namespace scream::mam_coupling { + +using view_1d_host = typename KT::view_1d::HostMirror; +using view_1d_int_host = typename KT::view_1d::HostMirror; +using view_2d_host = typename KT::view_2d::HostMirror; +// using view_5d_host = typename KT::view_ND::HostMirror; +// using complex_view_1d = typename KT::view_1d>; + +// constexpr int nlwbands = mam4::modal_aer_opt::nlwbands; +// constexpr int nswbands = mam4::modal_aer_opt::nswbands; + +struct AerosolSurfaceEmissionsHostData { + // these have dim = n_species + view_1d_host emis_species_index; + view_1d_host emis_species_units; + view_1d_host emis_species_name; + // molecular weight + view_1d_host emis_species_mw; + // number of sectors in each field + view_1d_int_host emis_species_nsectors; + // FIXME: not quite sure what this does--maybe just a placeholder for fields(:, i_sector)? + view_1d_host emis_species_sector; + // note fields have dim = n_species x nsectors + // TODO: fields have units??? maybe the same as the upper spec units + view_2d_host emis_species_fields; +}; + +using AerosolSurfaceEmissionsDeviceData = + mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; + +inline void set_emissions_params( + AerosolSurfaceEmissionsHostData &aerosol_emissions_host_data, + ekat::ParameterList ¶ms_emissions, + std::map &layouts, + std::map &host_views) { + // Set up input structure to read data from file. + using strvec_t = std::vector; + using namespace ShortFieldTagsNames; + + // using SrfEmisDims = mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; + // SrfEmisDims srf_emimssions_dims; + +} + +} // namespace scream::mam_coupling + +#endif From b30d5fdf25e7416817b00cb05f30ea36f1c197cc Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 11 Jul 2024 19:01:51 -0600 Subject: [PATCH 287/477] reading files but scorpio issues with freeing file --- ...and_online_emissions_process_interface.cpp | 398 +++++++++--------- ...and_online_emissions_process_interface.hpp | 37 +- .../src/physics/mam/mam_emissions_utils.hpp | 84 ++-- .../single-process/mam/emissions/input.yaml | 26 +- 4 files changed, 301 insertions(+), 244 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index b34186d7de82..3acf7e6f5a19 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -16,8 +16,8 @@ write in/outs for all variables clearly namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm& comm, - const ekat::ParameterList& params) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -35,22 +35,22 @@ void MAMSrfOnlineEmiss::set_grids( Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers + // NOTE: final output with be a flux for each grid point + // e.g., flux__emissions(Nx, Ny, Nspec) + // [kg m^-2 s^-1] or [# m^-2 s^-1] grid_ = grids_manager->get_grid("Physics"); - const auto& grid_name = grid_->name(); + const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + const FieldLayout scalar3d_layout_mid = grid_->get_3d_scalar_layout(true); + const FieldLayout scalar3d_layout_int = grid_->get_3d_scalar_layout(false); // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + const FieldLayout scalar2d_layout = grid_->get_2d_scalar_layout(); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process @@ -69,23 +69,24 @@ void MAMSrfOnlineEmiss::set_grids( "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice cloud water [kg/kg] wet - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char* int_mmr_field_name = + const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { @@ -96,13 +97,12 @@ void MAMSrfOnlineEmiss::set_grids( } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - // printf("%s \n", int_nmr_field_name); + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char* cld_mmr_field_name = + const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if (strlen(cld_mmr_field_name) > 0) { @@ -114,7 +114,7 @@ void MAMSrfOnlineEmiss::set_grids( // aerosol-related gases: mass mixing ratios for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } @@ -131,7 +131,7 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { // ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. -void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), "Error! Insufficient buffer size.\n"); @@ -143,7 +143,42 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } // ========================================================================================= +// // TODO: comments! +// void MAMSrfOnlineEmiss::set_emissions_names( +// const std::map map_spec_id, const std::string +// emis_type, const ekat::ParameterList &m_params, std::map &host_views) { + +// using view_1d_host = typename KT::view_1d::HostMirror; + +// // names take the form online_emis_specifier_for_ +// for (const auto &item : map_spec_id) { +// const auto spec_name = item.first; +// const int species_id = item.second; +// const auto file_name = emis_type + "_emis_specifier_" + spec_name; +// const auto &fname = m_params.get(file_name); + +// // read data +// AtmosphereInput srf_emissions_reader(m_params, grid_, +// host_views_emissions, +// layouts_emissions); +// srf_emissions_reader.read_variables(); +// srf_emissions_reader.finalize(); +// } // end ispec +// // for (int i = 0; i < nswbands; i++) { +// // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); +// // specrefndxsw_host(i, species_id).imag() = +// // haero::abs(host_views[sw_im_name](i)); +// // } +// // for (int i = 0; i < nlwbands; i++) { +// // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); +// // specrefndxlw_host(i, species_id).imag() = +// // haero::abs(host_views[lw_im_name](i)); +// // } + +// } // end set_emissions_names +// ========================================================================================= // inline void set_emissions_layouts( // const std::map map_spec_id, // const std::string emis_type, @@ -160,26 +195,24 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { // // const auto& fname = m_params.get(file_name); // // update file name - // // read data // AtmosphereInput srf_emissions_reader( -// params_srf_emissions, grid_, host_views_emissions, layouts_emissions); +// params_srf_emissions, grid_, host_views_emissions, +// layouts_emissions); // srf_emissions_reader.read_variables(); // srf_emissions_reader.finalize(); // } // end ispec - - - // for (int i = 0; i < nswbands; i++) { - // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); - // specrefndxsw_host(i, species_id).imag() = - // haero::abs(host_views[sw_im_name](i)); - // } - // for (int i = 0; i < nlwbands; i++) { - // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); - // specrefndxlw_host(i, species_id).imag() = - // haero::abs(host_views[lw_im_name](i)); - // } +// for (int i = 0; i < nswbands; i++) { +// specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); +// specrefndxsw_host(i, species_id).imag() = +// haero::abs(host_views[sw_im_name](i)); +// } +// for (int i = 0; i < nlwbands; i++) { +// specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); +// specrefndxlw_host(i, species_id).imag() = +// haero::abs(host_views[lw_im_name](i)); +// } // } // end // ========================================================================================= @@ -187,16 +220,17 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); dry_atm_.qv = buffer_.qv_dry; dry_atm_.qc = buffer_.qc_dry; dry_atm_.nc = buffer_.nc_dry; @@ -215,40 +249,41 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); + get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = + const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); + get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char* cld_mmr_field_name = + const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if (strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); + get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } @@ -256,148 +291,125 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - // // read data from file - { - // // using namespace ShortFieldTagsNames; - - using view_1d_host = typename KT::view_1d::HostMirror; - - mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; - - // { - // make a list of host views - std::map host_views_srf_emissions; - // defines layouts - std::map layouts_srf_emissions; - ekat::ParameterList params_srf_emissions; - std::string prefix_srf_emissions = "srf_emis_specifier_for_"; - - // // constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; - // // auto specrefndxsw_host = mam_coupling::complex_view_2d::HostMirror( - // // "specrefndxsw_host", nswbands_, maxd_aspectype); - - // // auto specrefndxlw_host = mam_coupling::complex_view_2d::HostMirror( - // // "specrefndxlw_host", nlwbands_, maxd_aspectype); - - std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; - map_srf_emiss_name_species_id["num_a1"] = 3; - map_srf_emiss_name_species_id["num_a2"] = 4; - map_srf_emiss_name_species_id["num_a4"] = 5; - map_srf_emiss_name_species_id["pom_a4"] = 6; - map_srf_emiss_name_species_id["so4_a1"] = 7; - map_srf_emiss_name_species_id["so4_a2"] = 8; - - std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; - map_online_emiss_name_species_id["num_a1"] = 3; - map_online_emiss_name_species_id["num_a2"] = 4; - map_online_emiss_name_species_id["num_a4"] = 5; - map_online_emiss_name_species_id["pom_a4"] = 6; - map_online_emiss_name_species_id["so4_a1"] = 7; - map_online_emiss_name_species_id["so4_a2"] = 8; - - // To create the input object, we need to set: - // 1) names (do during read loop) - // 2) params - // 3) host views - // 4) layouts - // set_emissions_names(surname_emissions, params_srf_emissions, - // host_views_emissions, layouts_emissions); - - // inline void set_emissions_names( - // const std::map map_spec_id, - // const std::string emis_type, - // std::map& host_views, - // mam_coupling::complex_view_2d::HostMirror& - // specrefndxsw_host, // complex refractive index for water visible - // mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { - - // // names take the form "online_emis_specifier_for_SO2" - // for (const auto& item : map_spec_id) { - // const auto spec_name = item.first; - // const int species_id = item.second; - // const auto file_name = emis_type + "_emis_specifier_" + spec_name; - // const auto& fname = m_params.get(file_name); - // update file name - - - // // read data - // AtmosphereInput srf_emissions_reader( - // params_srf_emissions, grid_, host_views_emissions, layouts_emissions); - // srf_emissions_reader.read_variables(); - // srf_emissions_reader.finalize(); - // } // end ispec - - - - // for (int i = 0; i < nswbands; i++) { - // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); - // specrefndxsw_host(i, species_id).imag() = - // haero::abs(host_views[sw_im_name](i)); - // } - // for (int i = 0; i < nlwbands; i++) { - // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); - // specrefndxlw_host(i, species_id).imag() = - // haero::abs(host_views[lw_im_name](i)); - // } - - // } // end - - // =============== - // Names - // =============== - - // names take the form _emis_specifier_for_ - // set_emissions_names("srf", params_srf_emissions, host_views_srf_emissions, layouts_srf_emissions); - // set_emissions_names("online", params_srf_emissions, host_views_online_emissions, layouts_online_emissions); - - // =============== - // Params - // =============== - using strvec_t = std::vector; - params_srf_emissions.set("Skip_Grid_Checks", true); - // params_srf_emissions.set("Field Names", {refindex_real_sw, refindex_im_sw, - // refindex_real_lw, refindex_im_lw}); - // params_srf_emissions.set("Filename", fname); - - // =============== - // Host Views - // =============== - using view_1d_host = typename KT::view_1d::HostMirror; - std::map host_views; - - // host_views[refindex_real_sw] = view_1d_host(refindex_real_sw, - // nswbands); host_views[refindex_im_sw] = view_1d_host(refindex_im_sw, - // nswbands); host_views[refindex_real_lw] = - // view_1d_host(refindex_real_lw, nlwbands); host_views[refindex_im_lw] = - // view_1d_host(refindex_im_lw, nlwbands); - - // =============== - // Layouts - // =============== - std::map layouts; - // FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; - // FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; - - // layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); - // layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); - // layouts.emplace(refindex_real_lw, scalar_refindex_lw_layout); - // layouts.emplace(refindex_im_lw, scalar_refindex_lw_layout); - - // reshape specrefndxsw_host and copy it to device - // mam4::modal_aer_opt::set_device_specrefindex( - // aerosol_optics_device_data_.specrefindex_sw, "short_wave", - // specrefndxsw_host); - // mam4::modal_aer_opt::set_device_specrefindex( - // aerosol_optics_device_data_.specrefindex_lw, "long_wave", - // specrefndxlw_host); - } -} + // read data from files + using view_1d_host = typename KT::view_1d::HostMirror; + using view_2d_host = typename KT::view_2d::HostMirror; + using strvec_t = std::vector; + + // mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + + std::map map_srf_emiss_name_species_id; + strvec_t srf_emiss_spec_names; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["num_a1"] = 3; + map_srf_emiss_name_species_id["num_a2"] = 4; + map_srf_emiss_name_species_id["num_a4"] = 5; + map_srf_emiss_name_species_id["pom_a4"] = 6; + map_srf_emiss_name_species_id["so4_a1"] = 7; + map_srf_emiss_name_species_id["so4_a2"] = 8; + // for (const auto &item : map_srf_emiss_name_species_id) { + // srf_emiss_spec_names.push_back(item.first); + // } + + std::map map_online_emiss_name_species_id; + strvec_t online_emiss_spec_names; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["num_a1"] = 3; + map_online_emiss_name_species_id["num_a2"] = 4; + map_online_emiss_name_species_id["num_a4"] = 5; + map_online_emiss_name_species_id["pom_a4"] = 6; + map_online_emiss_name_species_id["so4_a1"] = 7; + map_online_emiss_name_species_id["so4_a2"] = 8; + // for (const auto &item : map_srf_emiss_name_species_id) { + // online_emiss_spec_names.push_back(item.first); + // } + + // To create the input object, we need to: + // - set names of views + // - declare host views + // - initialize FieldLayouts + // - initialize params + + using namespace ShortFieldTagsNames; + + // make a list of host views, that holds nspec-dimensional views of + // srf/online emissions at a grid point + std::map host_views_srf_emiss; + // list of layouts of srf/online emissions views + // NOTE: these are the same, but it seems best (necessary?) to have + // equally-sized lists + std::map layouts_srf_emiss; + ekat::ParameterList params_srf_emiss; + std::string middle_name_emiss = "_emis_specifier_for_"; + + // TODO: break this out into a function in emissions_utils.hpp + // host_views_srf_emiss[sname] = view_1d_host(sname, mam_coupling::n_srf_emiss); + // layouts_srf_emiss.emplace(sname, scalar_srf_emiss_layout); + // for (const auto &item : map_srf_emiss_name_species_id) { + // online_emiss_spec_names.push_back(item.first); + // const auto oname = item.first; + // host_views_online_emiss[oname] = view_2d_host(oname, ) + // } + + // set names of views + const std::string srf_emiss_name = "surface_emissions"; + // const std::string online_emiss_name = "online_emissions"; + + // declare the host views + host_views_srf_emiss[srf_emiss_name] = view_1d_host(srf_emiss_name, 1); + // host_views_emissions[online_emiss_name] = + // view_1d_host(online_emiss_name, mam_coupling::n_online_emiss); + + // initialize and collect FieldLayouts + // layout for 2D (2d horiz == 1d flattened col index) scalar-valued variable + FieldLayout scalar_srf_emiss_layout{{COL}, {ncol_}, {srf_emiss_name}}; + // FieldLayout scalar_online_emiss_layout{ + // {CMP}, {mam_coupling::n_online_emiss}, {online_emiss_name}}; + layouts_srf_emiss.emplace(srf_emiss_name, scalar_srf_emiss_layout); + // layouts_emissions.emplace(online_emiss_name, scalar_online_emiss_layout); + + // initialize params + params_srf_emiss.set("Skip_Grid_Checks", true); + // these need to be the emission-type names from the map__emiss_name_species_id maps + // params_srf_emiss.set("Field Names", {srf_emiss_name}); + + // namelist entries take the form _emis_specifier_for_ + std::string emis_type = "srf"; + for (const auto &item : map_srf_emiss_name_species_id) { + const auto spec_name = item.first; + const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + spec_name; + const auto &fpath = m_params.get(file_name); + params_srf_emiss.set("Filename", fpath); + + // read data + AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); + srf_emissions_reader.read_variables(); + srf_emissions_reader.finalize(); + // copy data to device + + } // end ispec + emis_type = "online"; + // for (const auto &item : map_online_emiss_name_species_id) { + // const auto spec_name = item.first; + // const int species_id = item.second; + // const auto file_name = emis_type + middle_name_emiss + spec_name; + // const auto &fname = m_params.get(file_name); + // params_emissions.set("Filename", fname); + + // // read data + // AtmosphereInput online_emissions_reader( + // m_params, grid_, host_views_emissions, layouts_emissions); + // online_emissions_reader.read_variables(); + // online_emissions_reader.finalize(); + // // copy data to device + + // } // end ispec +} // end initialize_impl() // ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index d59f0df56c1f..0abd1a50e857 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -14,6 +14,15 @@ // #include #include +// TODO: determine when these may be necessary +// #ifndef KOKKOS_ENABLE_CUDA +// #define protected_except_cuda public +// #define private_except_cuda public +// #else +// #define protected_except_cuda protected +// #define private_except_cuda private +// #endif + namespace scream { // The process responsible for handling MAM4 surface and online emissions. The @@ -37,7 +46,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - public: +public: // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -52,8 +61,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::string name() const { return "mam_srf_online_emissions"; } // grid - void set_grids( - const std::shared_ptr grids_manager) override; + void + set_grids(const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -78,25 +87,25 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { const mam_coupling::AerosolState &wet_aero, const mam_coupling::DryAtmosphere &dry_atm, const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; + dry_atm_pre_ = dry_atm; dry_aero_pre_ = dry_aero; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index + const int i = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); - } // operator() + } // operator() // local variables for preprocess struct // number of horizontal columns and vertical levels @@ -106,14 +115,14 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMAci::Preprocess - private: +private: // preprocessing scratch pad Preprocess preprocess_; -}; // MAMSrfOnlineEmiss +}; // MAMSrfOnlineEmiss -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp index 807896f12392..8549b47dd8c5 100644 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -18,41 +18,55 @@ using view_2d_host = typename KT::view_2d::HostMirror; // using view_5d_host = typename KT::view_ND::HostMirror; // using complex_view_1d = typename KT::view_1d>; -// constexpr int nlwbands = mam4::modal_aer_opt::nlwbands; -// constexpr int nswbands = mam4::modal_aer_opt::nswbands; - -struct AerosolSurfaceEmissionsHostData { - // these have dim = n_species - view_1d_host emis_species_index; - view_1d_host emis_species_units; - view_1d_host emis_species_name; - // molecular weight - view_1d_host emis_species_mw; - // number of sectors in each field - view_1d_int_host emis_species_nsectors; - // FIXME: not quite sure what this does--maybe just a placeholder for fields(:, i_sector)? - view_1d_host emis_species_sector; - // note fields have dim = n_species x nsectors - // TODO: fields have units??? maybe the same as the upper spec units - view_2d_host emis_species_fields; -}; - -using AerosolSurfaceEmissionsDeviceData = - mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; - -inline void set_emissions_params( - AerosolSurfaceEmissionsHostData &aerosol_emissions_host_data, - ekat::ParameterList ¶ms_emissions, - std::map &layouts, - std::map &host_views) { - // Set up input structure to read data from file. - using strvec_t = std::vector; - using namespace ShortFieldTagsNames; - - // using SrfEmisDims = mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; - // SrfEmisDims srf_emimssions_dims; - -} +constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; +constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; + +using namespace ShortFieldTagsNames; + +// struct AerosolSurfaceEmissionsHostData { +// // these have dim = n_species +// view_1d_host emis_species_index; +// view_1d_host emis_species_units; +// view_1d_host emis_species_name; +// // molecular weight +// view_1d_host emis_species_mw; +// // number of sectors in each field +// view_1d_int_host emis_species_nsectors; +// // FIXME: not quite sure what this does--maybe just a placeholder for +// // fields(:, i_sector)? +// view_1d_host emis_species_sector; +// // note fields have dim = n_species x nsectors +// // TODO: fields have units??? maybe the same as the upper spec units +// view_2d_host emis_species_fields; +// }; + +// using AerosolSurfaceEmissionsDeviceData = + // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; + +// inline void set_emissions_params( +// AerosolSurfaceEmissionsHostData& aerosol_emissions_host_data, +// ekat::ParameterList& params_emissions, +// std::map& layouts, +// std::map& host_views) { +// // Set up input structure to read data from file. +// using strvec_t = std::vector; +// // using namespace ShortFieldTagsNames; + +// // using SrfEmisDims = +// // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; SrfEmisDims +// // srf_emimssions_dims; +// } + +// inline void set_emissions_names(const std::map map_spec_id, +// const std::string emis_type, +// const ekat::ParameterList& m_params, +// std::map& host_views) { + +// using view_1d_host = typename KT::view_1d::HostMirror; + +// std::string + +// } // end set_emissions_names } // namespace scream::mam_coupling diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 741cf2970320..30861b656e68 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -10,7 +10,29 @@ time_stepping: atmosphere_processes: atm_procs_list: [mam4_srf_online_emiss] - + mam4_srf_online_emiss: + # MAM4xx-Surface-Emissions + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + # MAM4xx-Online-Emissions + online_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_SOAG: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + + grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -30,7 +52,7 @@ initial_conditions: #we should get the following variables from other processes pbl_height : 1.0 - + # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From 79b88b7862c3ec769dfcea5151c88a08194fffea Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 18 Jul 2024 17:36:54 -0600 Subject: [PATCH 288/477] surface and online emissions data files successfull read in --- ...and_online_emissions_process_interface.cpp | 236 +++++++----------- .../src/physics/mam/mam_emissions_utils.hpp | 18 ++ .../single-process/mam/optics/CMakeLists.txt | 4 +- 3 files changed, 113 insertions(+), 145 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 3acf7e6f5a19..72171a035416 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -144,77 +144,11 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { } // ========================================================================================= // // TODO: comments! -// void MAMSrfOnlineEmiss::set_emissions_names( -// const std::map map_spec_id, const std::string -// emis_type, const ekat::ParameterList &m_params, std::map &host_views) { - -// using view_1d_host = typename KT::view_1d::HostMirror; - -// // names take the form online_emis_specifier_for_ -// for (const auto &item : map_spec_id) { -// const auto spec_name = item.first; -// const int species_id = item.second; -// const auto file_name = emis_type + "_emis_specifier_" + spec_name; -// const auto &fname = m_params.get(file_name); - -// // read data -// AtmosphereInput srf_emissions_reader(m_params, grid_, -// host_views_emissions, -// layouts_emissions); -// srf_emissions_reader.read_variables(); -// srf_emissions_reader.finalize(); -// } // end ispec - -// // for (int i = 0; i < nswbands; i++) { -// // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); -// // specrefndxsw_host(i, species_id).imag() = -// // haero::abs(host_views[sw_im_name](i)); -// // } -// // for (int i = 0; i < nlwbands; i++) { -// // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); -// // specrefndxlw_host(i, species_id).imag() = -// // haero::abs(host_views[lw_im_name](i)); -// // } - -// } // end set_emissions_names +// void MAMSrfOnlineEmiss::set_emissions_names() {} \\ end set_emissions_names() // ========================================================================================= -// inline void set_emissions_layouts( -// const std::map map_spec_id, -// const std::string emis_type, -// std::map& host_views, -// mam_coupling::complex_view_2d::HostMirror& -// specrefndxsw_host, // complex refractive index for water visible -// mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { - -// // names take the form "online_emis_specifier_for_SO2" -// for (const auto& item : map_spec_id) { -// const auto spec_name = item.first; -// const int species_id = item.second; -// const auto file_name = emis_type + "_emis_specifier_" + spec_name; -// // const auto& fname = m_params.get(file_name); -// // update file name - -// // read data -// AtmosphereInput srf_emissions_reader( -// params_srf_emissions, grid_, host_views_emissions, -// layouts_emissions); -// srf_emissions_reader.read_variables(); -// srf_emissions_reader.finalize(); -// } // end ispec - -// for (int i = 0; i < nswbands; i++) { -// specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); -// specrefndxsw_host(i, species_id).imag() = -// haero::abs(host_views[sw_im_name](i)); -// } -// for (int i = 0; i < nlwbands; i++) { -// specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); -// specrefndxlw_host(i, species_id).imag() = -// haero::abs(host_views[lw_im_name](i)); -// } - -// } // end +// inline void set_emissions_layouts() { + +// } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options @@ -294,12 +228,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // read data from files using view_1d_host = typename KT::view_1d::HostMirror; using view_2d_host = typename KT::view_2d::HostMirror; - using strvec_t = std::vector; - - // mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + // these probably belong in mam4xx std::map map_srf_emiss_name_species_id; - strvec_t srf_emiss_spec_names; map_srf_emiss_name_species_id["DMS"] = 0; map_srf_emiss_name_species_id["SO2"] = 1; map_srf_emiss_name_species_id["bc_a4"] = 2; @@ -309,12 +240,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { map_srf_emiss_name_species_id["pom_a4"] = 6; map_srf_emiss_name_species_id["so4_a1"] = 7; map_srf_emiss_name_species_id["so4_a2"] = 8; - // for (const auto &item : map_srf_emiss_name_species_id) { - // srf_emiss_spec_names.push_back(item.first); - // } + // these probably belong in mam4xx std::map map_online_emiss_name_species_id; - strvec_t online_emiss_spec_names; map_online_emiss_name_species_id["SO2"] = 0; map_online_emiss_name_species_id["SOAG"] = 1; map_online_emiss_name_species_id["bc_a4"] = 2; @@ -324,65 +252,52 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { map_online_emiss_name_species_id["pom_a4"] = 6; map_online_emiss_name_species_id["so4_a1"] = 7; map_online_emiss_name_species_id["so4_a2"] = 8; - // for (const auto &item : map_srf_emiss_name_species_id) { - // online_emiss_spec_names.push_back(item.first); - // } - - // To create the input object, we need to: - // - set names of views - // - declare host views - // - initialize FieldLayouts - // - initialize params using namespace ShortFieldTagsNames; - // make a list of host views, that holds nspec-dimensional views of - // srf/online emissions at a grid point - std::map host_views_srf_emiss; - // list of layouts of srf/online emissions views - // NOTE: these are the same, but it seems best (necessary?) to have - // equally-sized lists - std::map layouts_srf_emiss; ekat::ParameterList params_srf_emiss; + ekat::ParameterList params_online_emiss; std::string middle_name_emiss = "_emis_specifier_for_"; - // TODO: break this out into a function in emissions_utils.hpp - // host_views_srf_emiss[sname] = view_1d_host(sname, mam_coupling::n_srf_emiss); - // layouts_srf_emiss.emplace(sname, scalar_srf_emiss_layout); - // for (const auto &item : map_srf_emiss_name_species_id) { - // online_emiss_spec_names.push_back(item.first); - // const auto oname = item.first; - // host_views_online_emiss[oname] = view_2d_host(oname, ) - // } - - // set names of views - const std::string srf_emiss_name = "surface_emissions"; - // const std::string online_emiss_name = "online_emissions"; - - // declare the host views - host_views_srf_emiss[srf_emiss_name] = view_1d_host(srf_emiss_name, 1); - // host_views_emissions[online_emiss_name] = - // view_1d_host(online_emiss_name, mam_coupling::n_online_emiss); - - // initialize and collect FieldLayouts - // layout for 2D (2d horiz == 1d flattened col index) scalar-valued variable - FieldLayout scalar_srf_emiss_layout{{COL}, {ncol_}, {srf_emiss_name}}; - // FieldLayout scalar_online_emiss_layout{ - // {CMP}, {mam_coupling::n_online_emiss}, {online_emiss_name}}; - layouts_srf_emiss.emplace(srf_emiss_name, scalar_srf_emiss_layout); - // layouts_emissions.emplace(online_emiss_name, scalar_online_emiss_layout); - // initialize params params_srf_emiss.set("Skip_Grid_Checks", true); - // these need to be the emission-type names from the map__emiss_name_species_id maps - // params_srf_emiss.set("Field Names", {srf_emiss_name}); + params_online_emiss.set("Skip_Grid_Checks", true); + + // FIXME: this, and all places used, will need to change to a flattened column index + int numlat_srf = scream::mam_coupling::nlat_srf; + int numlon_srf = scream::mam_coupling::nlon_srf; + int numcol_fake_srf = numlat_srf * numlon_srf; + const auto srf_emiss_var_names = mam4::mo_srf_emissions::srf_emimssions_data_fields; + view_2d_host srf_emiss_host; + + // TODO: break this out into a function in emissions_utils.hpp // namelist entries take the form _emis_specifier_for_ std::string emis_type = "srf"; for (const auto &item : map_srf_emiss_name_species_id) { - const auto spec_name = item.first; - const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + spec_name; + const std::string srf_emiss_name = item.first; + // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 + // Is this correct or an error in the data? + if (srf_emiss_name == "DMS") { + numlat_srf = 180; + numlon_srf = 360; + numcol_fake_srf = numlat_srf * numlon_srf; + } else { + numlat_srf = scream::mam_coupling::nlat_srf; + numlon_srf = scream::mam_coupling::nlon_srf; + numcol_fake_srf = numlat_srf * numlon_srf; + } + std::map host_views_srf_emiss; + std::map layouts_srf_emiss; + // FIXME: this will need to change to a flattened column index + FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, {"lat", "lon"}); + for ( const auto &var_name : srf_emiss_var_names.at(srf_emiss_name) ) { + host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); + layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); + } + // const auto spec_name = item.first; + // const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; const auto &fpath = m_params.get(file_name); params_srf_emiss.set("Filename", fpath); @@ -390,25 +305,60 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); srf_emissions_reader.read_variables(); srf_emissions_reader.finalize(); - // copy data to device - - } // end ispec + // copy data into host view for sending, columnwise, to mam4xx's mo_srf_emissions + // TODO: any reason not to just copy to device here? + // TODO: Could probably be a parfor here + // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { + // srf_emiss_host(colidx_fake, species_id) = host_views_srf_emiss[srf_emiss_name](colidx_fake); + // } + } // end item + + // FIXME: this, and all places used, will need to change to a flattened column index + const int numlat_online = scream::mam_coupling::nlat_online; + const int numlon_online = scream::mam_coupling::nlon_online; + int numalti_online = scream::mam_coupling::nalti_online; + int numcol_fake_online = numlat_online * numlon_online * numalti_online; + const auto online_emiss_var_names = mam4::aero_model_emissions::online_emimssions_data_fields; + + // view_3d_host online_emiss_host; emis_type = "online"; - // for (const auto &item : map_online_emiss_name_species_id) { - // const auto spec_name = item.first; - // const int species_id = item.second; - // const auto file_name = emis_type + middle_name_emiss + spec_name; - // const auto &fname = m_params.get(file_name); - // params_emissions.set("Filename", fname); - - // // read data - // AtmosphereInput online_emissions_reader( - // m_params, grid_, host_views_emissions, layouts_emissions); - // online_emissions_reader.read_variables(); - // online_emissions_reader.finalize(); - // // copy data to device - - // } // end ispec + for (const auto &item : map_online_emiss_name_species_id) { + const std::string online_emiss_name = item.first; + // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 + // Is this correct or an error in the data? + if (online_emiss_name == "SOAG") { + numalti_online = 12; + numcol_fake_online = numlat_online * numlon_online * numalti_online; + } else { + numalti_online = 13; + numcol_fake_online = numlat_online * numlon_online * numalti_online; + } + std::map host_views_online_emiss; + std::map layouts_online_emiss; + // FIXME: this will need to change to a flattened column index + FieldLayout scalar_online_emiss_layout({CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, {"altitude", "lat", "lon"}); + for ( const auto &var_name : online_emiss_var_names.at(online_emiss_name) ) { + host_views_online_emiss[var_name] = view_1d_host(var_name, numcol_fake_online); + layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); + } // end var_name + // const auto spec_name = item.first; + // const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + online_emiss_name; + const auto &fpath = m_params.get(file_name); + params_online_emiss.set("Filename", fpath); + + // read data + AtmosphereInput online_emissions_reader(params_online_emiss, grid_, host_views_online_emiss, layouts_online_emiss); + online_emissions_reader.read_variables(); + online_emissions_reader.finalize(); + // copy data into host view for sending, columnwise, to mam4xx's mo_online_emissions + // TODO: any reason not to just copy to device here? + // TODO: Could probably be a parfor here + // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { + // online_emiss_host(colidx_fake_online, species_id) = host_views_online_emiss[online_emiss_name](colidx_fake); + // } + } // end item + } // end initialize_impl() // ============================================================================= diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp index 8549b47dd8c5..527efd5de18b 100644 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -21,8 +21,26 @@ using view_2d_host = typename KT::view_2d::HostMirror; constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; +// FIXME: this will need to change when we remap to flattened column idx +constexpr int nlat_srf = 96; +constexpr int nlon_srf = 144; + +constexpr int nalti_online = 13; +constexpr int nlat_online = 96; +constexpr int nlon_online = 144; + using namespace ShortFieldTagsNames; +// std::map> map_srf_emiss_file_vars; + +inline void set_file_var_names(std::map> &var_map, + std::map &spec_map) { + // for (const auto &spec : spec_map) { + // std::string spec_name = spec.first; + // std::cout << "var_map[spec_name] = " << var_map[spec_name] << "\n"; + // } +} + // struct AerosolSurfaceEmissionsHostData { // // these have dim = n_species // view_1d_host emis_species_index; diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt index a3e6f64ec7d0..50f2cfa68a3b 100644 --- a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -40,7 +40,7 @@ set (TEST_INPUT_FILES scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc scream/mam4xx/physprops/poly_rrtmg_c20240206.nc - scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + # scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) @@ -63,4 +63,4 @@ if (SCREAM_ENABLE_BASELINE_TESTS) # Note: one is enough, since we already check that np1 is BFB with npX set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) -endif() \ No newline at end of file +endif() From f5da2455281e8cd2d19c3bb146d7aa367c45e761 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 07:21:43 -0700 Subject: [PATCH 289/477] Horizontal remapper create_horiz_remapper works with so2 srf file --- .../mam4xx/srf_online_emiss/shell_commands | 13 ++ ...and_online_emissions_process_interface.cpp | 176 ++++++++++-------- ...and_online_emissions_process_interface.hpp | 37 ++-- .../eamxx/src/physics/mam/srf_emission.hpp | 22 +++ .../src/physics/mam/srf_emission_impl.hpp | 86 +++++++++ 5 files changed, 244 insertions(+), 90 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands create mode 100644 components/eamxx/src/physics/mam/srf_emission.hpp create mode 100644 components/eamxx/src/physics/mam/srf_emission_impl.hpp diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands new file mode 100644 index 000000000000..6995cd8b8562 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands @@ -0,0 +1,13 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +#modify initial condition file to get aerosol species ICs +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b + +# Add spa as RRTMG needs spa +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 72171a035416..a694e63a5e09 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -33,16 +33,16 @@ void MAMSrfOnlineEmiss::set_grids( // Nevertheless, for output reasons, we like to see 'kg/kg'. Units q_unit(kg / kg, "kg/kg"); - Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers + Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers // NOTE: final output with be a flux for each grid point // e.g., flux__emissions(Nx, Ny, Nspec) // [kg m^-2 s^-1] or [# m^-2 s^-1] - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces @@ -56,56 +56,56 @@ void MAMSrfOnlineEmiss::set_grids( // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] + grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] + grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure + grid_name); // total pressure add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] + grid_name); // pseudo density in [Pa] add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity + "tracers"); // specific humidity add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet + "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet + "tracers"); // ice cloud water [kg/kg] wet add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio + "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints + grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio + "tracers"); // cloud liquid wet number mixing ratio // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { + if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { + if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } @@ -113,11 +113,21 @@ void MAMSrfOnlineEmiss::set_grids( } // aerosol-related gases: mass mixing ratios - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } + + // Reading so2 srf emiss data + + std::string so2_data_file = + "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" + "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; + std::string srf_map_file = ""; + // Init horizontal remap + srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( + grid_, so2_data_file, srf_map_file, m_iop != nullptr); } // ========================================================================================= @@ -132,9 +142,9 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= - requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); @@ -181,7 +191,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = @@ -194,11 +204,11 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { + if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; @@ -207,14 +217,14 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { + if(strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); @@ -231,9 +241,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // these probably belong in mam4xx std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; map_srf_emiss_name_species_id["num_a1"] = 3; map_srf_emiss_name_species_id["num_a2"] = 4; map_srf_emiss_name_species_id["num_a4"] = 5; @@ -243,9 +253,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // these probably belong in mam4xx std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; map_online_emiss_name_species_id["num_a1"] = 3; map_online_emiss_name_species_id["num_a2"] = 4; map_online_emiss_name_species_id["num_a4"] = 5; @@ -263,107 +273,121 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { params_srf_emiss.set("Skip_Grid_Checks", true); params_online_emiss.set("Skip_Grid_Checks", true); - // FIXME: this, and all places used, will need to change to a flattened column index - int numlat_srf = scream::mam_coupling::nlat_srf; - int numlon_srf = scream::mam_coupling::nlon_srf; + // FIXME: this, and all places used, will need to change to a flattened column + // index + int numlat_srf = scream::mam_coupling::nlat_srf; + int numlon_srf = scream::mam_coupling::nlon_srf; int numcol_fake_srf = numlat_srf * numlon_srf; - const auto srf_emiss_var_names = mam4::mo_srf_emissions::srf_emimssions_data_fields; + const auto srf_emiss_var_names = + mam4::mo_srf_emissions::srf_emimssions_data_fields; view_2d_host srf_emiss_host; // TODO: break this out into a function in emissions_utils.hpp // namelist entries take the form _emis_specifier_for_ std::string emis_type = "srf"; - for (const auto &item : map_srf_emiss_name_species_id) { + for(const auto &item : map_srf_emiss_name_species_id) { const std::string srf_emiss_name = item.first; // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 // Is this correct or an error in the data? - if (srf_emiss_name == "DMS") { - numlat_srf = 180; - numlon_srf = 360; + if(srf_emiss_name == "DMS") { + numlat_srf = 180; + numlon_srf = 360; numcol_fake_srf = numlat_srf * numlon_srf; } else { - numlat_srf = scream::mam_coupling::nlat_srf; - numlon_srf = scream::mam_coupling::nlon_srf; + numlat_srf = scream::mam_coupling::nlat_srf; + numlon_srf = scream::mam_coupling::nlon_srf; numcol_fake_srf = numlat_srf * numlon_srf; } std::map host_views_srf_emiss; std::map layouts_srf_emiss; // FIXME: this will need to change to a flattened column index - FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, {"lat", "lon"}); - for ( const auto &var_name : srf_emiss_var_names.at(srf_emiss_name) ) { + FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, + {"lat", "lon"}); + for(const auto &var_name : srf_emiss_var_names.at(srf_emiss_name)) { host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); } // const auto spec_name = item.first; // const int species_id = item.second; const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; - const auto &fpath = m_params.get(file_name); + const auto &fpath = m_params.get(file_name); params_srf_emiss.set("Filename", fpath); // read data - AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); - srf_emissions_reader.read_variables(); - srf_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's mo_srf_emissions + /// AtmosphereInput srf_emissions_reader( + // params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); + // srf_emissions_reader.read_variables(); + // srf_emissions_reader.finalize(); + // copy data into host view for sending, columnwise, to mam4xx's + // mo_srf_emissions // TODO: any reason not to just copy to device here? // TODO: Could probably be a parfor here // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { - // srf_emiss_host(colidx_fake, species_id) = host_views_srf_emiss[srf_emiss_name](colidx_fake); + // srf_emiss_host(colidx_fake, species_id) = + // host_views_srf_emiss[srf_emiss_name](colidx_fake); // } - } // end item + } // end item - // FIXME: this, and all places used, will need to change to a flattened column index - const int numlat_online = scream::mam_coupling::nlat_online; + // FIXME: this, and all places used, will need to change to a flattened column + // index + /*const int numlat_online = scream::mam_coupling::nlat_online; const int numlon_online = scream::mam_coupling::nlon_online; - int numalti_online = scream::mam_coupling::nalti_online; - int numcol_fake_online = numlat_online * numlon_online * numalti_online; - const auto online_emiss_var_names = mam4::aero_model_emissions::online_emimssions_data_fields; + int numalti_online = scream::mam_coupling::nalti_online; + int numcol_fake_online = numlat_online * numlon_online * numalti_online; + const auto online_emiss_var_names = + mam4::aero_model_emissions::online_emimssions_data_fields; // view_3d_host online_emiss_host; emis_type = "online"; - for (const auto &item : map_online_emiss_name_species_id) { + for(const auto &item : map_online_emiss_name_species_id) { const std::string online_emiss_name = item.first; // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 // Is this correct or an error in the data? - if (online_emiss_name == "SOAG") { - numalti_online = 12; + if(online_emiss_name == "SOAG") { + numalti_online = 12; numcol_fake_online = numlat_online * numlon_online * numalti_online; } else { - numalti_online = 13; + numalti_online = 13; numcol_fake_online = numlat_online * numlon_online * numalti_online; } std::map host_views_online_emiss; std::map layouts_online_emiss; // FIXME: this will need to change to a flattened column index - FieldLayout scalar_online_emiss_layout({CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, {"altitude", "lat", "lon"}); - for ( const auto &var_name : online_emiss_var_names.at(online_emiss_name) ) { - host_views_online_emiss[var_name] = view_1d_host(var_name, numcol_fake_online); + FieldLayout scalar_online_emiss_layout( + {CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, + {"altitude", "lat", "lon"}); + for(const auto &var_name : online_emiss_var_names.at(online_emiss_name)) { + host_views_online_emiss[var_name] = + view_1d_host(var_name, numcol_fake_online); layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); - } // end var_name + } // end var_name // const auto spec_name = item.first; // const int species_id = item.second; const auto file_name = emis_type + middle_name_emiss + online_emiss_name; - const auto &fpath = m_params.get(file_name); + const auto &fpath = m_params.get(file_name); params_online_emiss.set("Filename", fpath); // read data - AtmosphereInput online_emissions_reader(params_online_emiss, grid_, host_views_online_emiss, layouts_online_emiss); + AtmosphereInput online_emissions_reader(params_online_emiss, grid_, + host_views_online_emiss, + layouts_online_emiss); online_emissions_reader.read_variables(); online_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's mo_online_emissions + // copy data into host view for sending, columnwise, to mam4xx's + // mo_online_emissions // TODO: any reason not to just copy to device here? // TODO: Could probably be a parfor here // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { - // online_emiss_host(colidx_fake_online, species_id) = host_views_online_emiss[online_emiss_name](colidx_fake); + // online_emiss_host(colidx_fake_online, species_id) = + // host_views_online_emiss[online_emiss_name](colidx_fake); // } - } // end item - -} // end initialize_impl() + } // end item +*/ +} // end initialize_impl() // ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -411,4 +435,4 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { } // ============================================================================= -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 0abd1a50e857..c08aac75a9c6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -7,6 +7,7 @@ #include // For MAM4 aerosol configuration #include +#include #include // For declaring surface and online emission class derived from atm process // class @@ -46,7 +47,15 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; -public: + // Structures to store the data used for interpolation + std::shared_ptr srfEmissHorizInterp_; + + public: + using srfEmissFunc = mam_coupling::srfEmissFunctions; + + template + using uview_2d = Unmanaged>; + // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -61,8 +70,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::string name() const { return "mam_srf_online_emissions"; } // grid - void - set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -87,25 +96,25 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { const mam_coupling::AerosolState &wet_aero, const mam_coupling::DryAtmosphere &dry_atm, const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; + dry_atm_pre_ = dry_atm; dry_aero_pre_ = dry_aero; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index + const int i = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); - } // operator() + } // operator() // local variables for preprocess struct // number of horizontal columns and vertical levels @@ -115,14 +124,14 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMAci::Preprocess -private: + private: // preprocessing scratch pad Preprocess preprocess_; -}; // MAMSrfOnlineEmiss +}; // MAMSrfOnlineEmiss -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp new file mode 100644 index 000000000000..2d100f42b5c1 --- /dev/null +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -0,0 +1,22 @@ +#ifndef SRF_EMISSION_HPP +#define SRF_EMISSION_HPP + +namespace scream::mam_coupling { +namespace { + +template +struct srfEmissFunctions { + /* ------------------------------------------------------------------------------------------- + */ + // Surface emissions routines + + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &spa_data_file, const std::string &map_file, + const bool use_iop = false); +}; // struct srfEmissFunctions +} // namespace +} // namespace scream::mam_coupling +#endif // SRF_EMISSION_HPP + +#include "srf_emission_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp new file mode 100644 index 000000000000..8bd9465442c3 --- /dev/null +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -0,0 +1,86 @@ +#ifndef SRF_EMISSION_IMPL_HPP +#define SRF_EMISSION_IMPL_HPP + +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" + +namespace scream::mam_coupling { +namespace { + +template +std::shared_ptr +srfEmissFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const bool use_iop) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model or + use_iop /*IOP class defines it's own remapper for file data*/) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen spa data to fit the " + "model. We only allow\n" + " spa data to be at the same or coarser resolution " + "as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG( + map_file != "", + "ERROR: Spa data is on a different grid than the model one,\n" + " but spa_remap_file is missing from SPA parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + Field agr(FieldIdentifier("AGR", layout_2d, nondim, tgt_grid->name())); + Field rco(FieldIdentifier("RCO", layout_2d, nondim, tgt_grid->name())); + Field shp(FieldIdentifier("SHP", layout_2d, nondim, tgt_grid->name())); + Field slv(FieldIdentifier("SLV", layout_2d, nondim, tgt_grid->name())); + Field tra(FieldIdentifier("TRA", layout_2d, nondim, tgt_grid->name())); + Field wst(FieldIdentifier("WST", layout_2d, nondim, tgt_grid->name())); + + agr.allocate_view(); + rco.allocate_view(); + shp.allocate_view(); + slv.allocate_view(); + tra.allocate_view(); + wst.allocate_view(); + + remapper->register_field_from_tgt(agr); + remapper->register_field_from_tgt(rco); + remapper->register_field_from_tgt(shp); + remapper->register_field_from_tgt(slv); + remapper->register_field_from_tgt(tra); + remapper->register_field_from_tgt(wst); + + remapper->registration_ends(); + + return remapper; +} +} // namespace +} // namespace scream::mam_coupling + +#endif // SRF_EMISSION_IMPL_HPP \ No newline at end of file From 261ea3962dc525ba0064ae3edb4da6a9b01df1c7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 12:15:32 -0700 Subject: [PATCH 290/477] Adds srfmiss input --- ...and_online_emissions_process_interface.cpp | 5 +++ ...and_online_emissions_process_interface.hpp | 4 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 40 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a694e63a5e09..16fdd02d9818 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -128,6 +128,11 @@ void MAMSrfOnlineEmiss::set_grids( // Init horizontal remap srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, srf_map_file, m_iop != nullptr); + + // 2. Initialize the size of the SPAData structures. + srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); + srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); + // SrfEmissData_out_.init(ncol_, false); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index c08aac75a9c6..6a4923d1a7d7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -130,6 +130,10 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; + srfEmissFunc::srfEmissInput srfEmissData_start_; + srfEmissFunc::srfEmissInput srfEmissData_end_; + // srfEmissFunc::srfEmissOutput srfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 2d100f42b5c1..875a82d68319 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -6,6 +6,46 @@ namespace { template struct srfEmissFunctions { + struct srfEmissData { + srfEmissData() = default; + srfEmissData(const int ncol_) { init(ncol_, true); } + + void init(const int ncol_, const bool allocate) { + ncols = ncol_; + + if(allocate) { + AGR = view_1d("", ncols); + RCO = view_1d("", ncols); + SHP = view_1d("", ncols); + SLV = view_1d("", ncols); + TRA = view_1d("", ncols); + WST = view_1d("", ncols); + } + } + + // Basic spatial dimensions of the data + int ncols; + + view_1d AGR; + view_1d RCO; + view_1d SHP; + view_1d SLV; + view_1d TRA; + view_1d WST; + }; // srfEmissData + + struct srfEmissInput { + srfEmissInput() = default; + srfEmissInput(const int ncols_) { init(ncols_); } + + void init(const int ncols_) { data.init(ncols_, true); } + srfEmissData data; // All srfEmiss fields + }; // srfEmissInput + + // The output is really just srfEmissData, but for clarity it might + // help to see a srfEmissOutput along a srfEmissInput in functions signatures + using srfEmissOutput = srfEmissData; + /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines From cb2890266e7bba227279233e60834db500ddb4d7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 12:51:54 -0700 Subject: [PATCH 291/477] Adds a call to create_srfEmiss_data_reader --- ...srf_and_online_emissions_process_interface.cpp | 7 ++++++- ...srf_and_online_emissions_process_interface.hpp | 5 ++++- components/eamxx/src/physics/mam/srf_emission.hpp | 5 +++++ .../eamxx/src/physics/mam/srf_emission_impl.hpp | 15 +++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 16fdd02d9818..50ac5d151773 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -132,7 +132,12 @@ void MAMSrfOnlineEmiss::set_grids( // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - // SrfEmissData_out_.init(ncol_, false); + srfEmissData_out_.init(ncol_, false); + // 3. Skip as we don't need vertical interpolation + // 4. Create reader for srfEmiss data. The reader is an + // AtmosphereInput object + srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( + srfEmissHorizInterp_, so2_data_file); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 6a4923d1a7d7..145655d18525 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -130,9 +130,12 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; + // IO structure to read in data for standard grids + std::shared_ptr srfEmissDataReader_; + srfEmissFunc::srfEmissInput srfEmissData_start_; srfEmissFunc::srfEmissInput srfEmissData_end_; - // srfEmissFunc::srfEmissOutput srfEmissData_out_; + srfEmissFunc::srfEmissOutput srfEmissData_out_; }; // MAMSrfOnlineEmiss diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 875a82d68319..9efb84172730 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -54,6 +54,11 @@ struct srfEmissFunctions { const std::shared_ptr &model_grid, const std::string &spa_data_file, const std::string &map_file, const bool use_iop = false); + + static std::shared_ptr create_srfEmiss_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &srfEmiss_data_file); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 8bd9465442c3..1f2c8052eef2 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -79,7 +79,22 @@ srfEmissFunctions::create_horiz_remapper( remapper->registration_ends(); return remapper; +} // create_horiz_remapper + +template +std::shared_ptr +srfEmissFunctions::create_srfEmiss_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &srfEmiss_data_file) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(srfEmiss_data_file, io_grid, + io_fields, true); } + } // namespace } // namespace scream::mam_coupling From 7a37ccf507e565f06a87267131097fe179b6ce07 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 19:09:33 -0700 Subject: [PATCH 292/477] update_srfEmiss_data_from_file call in initialize_impl works --- ...and_online_emissions_process_interface.cpp | 8 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 8 ++ .../src/physics/mam/srf_emission_impl.hpp | 81 +++++++++++++++++-- 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 50ac5d151773..aa2f1d0bae33 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -241,6 +241,14 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // Load the first month into srfEmiss_end. + // Note: At the first time step, the data will be moved into srfEmiss_beg, + // and srfEmiss_end will be reloaded from file with the new month. + const int curr_month = timestamp().get_month() - 1; // 0-based + srfEmissFunc::update_srfEmiss_data_from_file( + srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, + srfEmissData_end_); + // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 9efb84172730..7fe5e9777921 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -1,6 +1,8 @@ #ifndef SRF_EMISSION_HPP #define SRF_EMISSION_HPP +#include "share/util/scream_timing.hpp" + namespace scream::mam_coupling { namespace { @@ -59,6 +61,12 @@ struct srfEmissFunctions { const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); + static void update_srfEmiss_data_from_file( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, + const int time_index, // zero-based + AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 1f2c8052eef2..f835e7354591 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -33,16 +33,18 @@ srfEmissFunctions::create_horiz_remapper( remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { - EKAT_REQUIRE_MSG(ncols_data <= ncols_model, - "Error! We do not allow to coarsen spa data to fit the " - "model. We only allow\n" - " spa data to be at the same or coarser resolution " - "as the model.\n"); + EKAT_REQUIRE_MSG( + ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit the " + "model. We only allow\n" + " srfEmiss data to be at the same or coarser resolution " + "as the model.\n"); // We must have a valid map file EKAT_REQUIRE_MSG( map_file != "", - "ERROR: Spa data is on a different grid than the model one,\n" - " but spa_remap_file is missing from SPA parameter list."); + "ERROR: srfEmiss data is on a different grid than the model one,\n" + " but srfEmiss_remap_file is missing from srfEmiss parameter " + "list."); remapper = std::make_shared(horiz_interp_tgt_grid, map_file); @@ -95,6 +97,71 @@ srfEmissFunctions::create_srfEmiss_data_reader( io_fields, true); } +template +void srfEmissFunctions::update_srfEmiss_data_from_file( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + const int time_index, // zero-based + AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { + using namespace ShortFieldTagsNames; + using ESU = ekat::ExeSpaceUtils; + using Member = typename KokkosTypes::MemberType; + + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); + + // 1. Read from file + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::read_data"); + scorpio_reader->read_variables(time_index); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::read_data"); + + // 2. Run the horiz remapper (it is a do-nothing op if srfEmiss data is on + // same grid as model) + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::horiz_remap"); + srfEmiss_horiz_interp.remap(/*forward = */ true); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::horiz_remap"); + + // 3. Copy from the tgt field of the remapper into the srfEmiss_data, padding + // data if necessary + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); + // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, + // tau_sw, tau_lw + + auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); + auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); + auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); + auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); + auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); + auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); + + const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) + .get_header() + .get_identifier() + .get_layout(); + + const int ncols = layout.dim(COL); + + auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); + auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); + auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); + auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); + auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); + auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); + + auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { + int icol = team.league_rank(); + srfEmiss_data_agr(icol) = agr(icol); + srfEmiss_data_rco(icol) = rco(icol); + srfEmiss_data_shp(icol) = shp(icol); + srfEmiss_data_slv(icol) = slv(icol); + srfEmiss_data_tra(icol) = tra(icol); + srfEmiss_data_wst(icol) = wst(icol); + }; + Kokkos::fence(); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); + + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); + +} // END update_srfEmiss_data_from_file + } // namespace } // namespace scream::mam_coupling From 7b2c9b257863b243f02186500e1e6a875be9f665 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 19:44:37 -0700 Subject: [PATCH 293/477] enables update_srfEmiss_timestate in run_impl --- ...and_online_emissions_process_interface.cpp | 15 ++++++++ ...and_online_emissions_process_interface.hpp | 2 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 22 +++++++++++- .../src/physics/mam/srf_emission_impl.hpp | 34 +++++++++++++++++++ 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index aa2f1d0bae33..06197e27ce49 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -413,6 +413,21 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + // Gather time and state information for interpolation + auto ts = timestamp() + dt; + // Update the srfEmissTimeState to reflect the current time, note the addition + // of dt + srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + srfEmissDataReader_, ts, *srfEmissHorizInterp_, srfEmissTimeState_, + srfEmissData_start_, srfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + // const auto& pmid_tgt = get_field_in("p_mid").get_view(); + // srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start, + // srfEmissData_end,m_buffer.srfEmiss_temp,srfEmissData_out); + /* Rough notes: Here we should implement or port the chem_emissions subroutine in diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 145655d18525..1add75173b8f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -133,6 +133,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // IO structure to read in data for standard grids std::shared_ptr srfEmissDataReader_; + srfEmissFunc::srfEmissTimeState srfEmissTimeState_; + srfEmissFunc::srfEmissInput srfEmissData_start_; srfEmissFunc::srfEmissInput srfEmissData_end_; srfEmissFunc::srfEmissOutput srfEmissData_out_; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 7fe5e9777921..f0b1bb6795f8 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -8,6 +8,21 @@ namespace { template struct srfEmissFunctions { + struct srfEmissTimeState { + srfEmissTimeState() = default; + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; + }; // srfEmissTimeState + struct srfEmissData { srfEmissData() = default; srfEmissData(const int ncol_) { init(ncol_, true); } @@ -54,7 +69,7 @@ struct srfEmissFunctions { static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &spa_data_file, const std::string &map_file, + const std::string &srfEmiss_data_file, const std::string &map_file, const bool use_iop = false); static std::shared_ptr create_srfEmiss_data_reader( @@ -66,6 +81,11 @@ struct srfEmissFunctions { const util::TimeStamp &ts, const int time_index, // zero-based AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input); + static void update_srfEmiss_timestate( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, AbstractRemapper &srfEmiss_horiz_interp, + srfEmissTimeState &time_state, srfEmissInput &srfEmiss_beg, + srfEmissInput &srfEmiss_end); }; // struct srfEmissFunctions } // namespace diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index f835e7354591..1adadaa0611c 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -162,6 +162,40 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( } // END update_srfEmiss_data_from_file +template +void srfEmissFunctions::update_srfEmiss_timestate( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + AbstractRemapper &srfEmiss_horiz_interp, srfEmissTimeState &time_state, + srfEmissInput &srfEmiss_beg, srfEmissInput &srfEmiss_end) { + // Now we check if we have to update the data that changes monthly + // NOTE: This means that srfEmiss assumes monthly data to update. Not + // any other frequency. + const auto month = ts.get_month() - 1; // Make it 0-based + if(month != time_state.current_month) { + // Update the srfEmiss time state information + time_state.current_month = month; + time_state.t_beg_month = + util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) + .frac_of_year_in_days(); + time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + + // Copy srfEmiss_end'data into srfEmiss_beg'data, and read in the new + // srfEmiss_end + std::swap(srfEmiss_beg, srfEmiss_end); + + // Update the srfEmiss forcing data for this month and next month + // Start by copying next months data to this months data structure. + // NOTE: If the timestep is bigger than monthly this could cause the wrong + // values + // to be assigned. A timestep greater than a month is very unlikely + // so we will proceed. + int next_month = (time_state.current_month + 1) % 12; + update_srfEmiss_data_from_file(scorpio_reader, ts, next_month, + srfEmiss_horiz_interp, srfEmiss_end); + } + +} // END updata_srfEmiss_timestate + } // namespace } // namespace scream::mam_coupling From 0827262f7c17e7eb3021e84f17b1f68b8ffe046e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 22:23:14 -0700 Subject: [PATCH 294/477] Implements the main routine to do the interpolation in run_impl --- ...and_online_emissions_process_interface.cpp | 7 +- ...and_online_emissions_process_interface.hpp | 3 + .../eamxx/src/physics/mam/srf_emission.hpp | 18 ++++ .../src/physics/mam/srf_emission_impl.hpp | 102 +++++++++++++++++- .../src/share/io/scream_scorpio_interface.cpp | 6 +- 5 files changed, 129 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 06197e27ce49..94978b773ddf 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -132,7 +132,7 @@ void MAMSrfOnlineEmiss::set_grids( // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_out_.init(ncol_, false); + srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? // 3. Skip as we don't need vertical interpolation // 4. Create reader for srfEmiss data. The reader is an // AtmosphereInput object @@ -425,8 +425,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. // const auto& pmid_tgt = get_field_in("p_mid").get_view(); - // srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start, - // srfEmissData_end,m_buffer.srfEmiss_temp,srfEmissData_out); + srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, + srfEmissData_end_, srfEmiss_temp_, + srfEmissData_out_); /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1add75173b8f..7fd5c54028f2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -139,6 +139,9 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { srfEmissFunc::srfEmissInput srfEmissData_end_; srfEmissFunc::srfEmissOutput srfEmissData_out_; + // Used to store temporary data during srfEmiss_main + srfEmissFunc::srfEmissInput srfEmiss_temp_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index f0b1bb6795f8..55980fabceb0 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -76,6 +76,12 @@ struct srfEmissFunctions { const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); + static void srfEmiss_main(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissInput &data_tmp, // Temporary + const srfEmissOutput &data_out); + static void update_srfEmiss_data_from_file( std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, @@ -87,6 +93,18 @@ struct srfEmissFunctions { srfEmissTimeState &time_state, srfEmissInput &srfEmiss_beg, srfEmissInput &srfEmiss_end); + // The following three are called during srfEmiss_main + static void perform_time_interpolation(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissOutput &data_out); + + // Performs convex interpolation of x0 and x1 at point t + template + KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 1adadaa0611c..a54abe168d2f 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -95,7 +95,107 @@ srfEmissFunctions::create_srfEmiss_data_reader( const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, io_fields, true); -} +} // create_srfEmiss_data_reader + +template +template +KOKKOS_INLINE_FUNCTION ScalarX srfEmissFunctions::linear_interp( + const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { + return (1 - t) * x0 + t * x1; +} // linear_interp + +template +void srfEmissFunctions::perform_time_interpolation( + const srfEmissTimeState &time_state, const srfEmissInput &data_beg, + const srfEmissInput &data_end, const srfEmissOutput &data_out) { + // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. + // IF this ever ceases to be the case, you can interp those too. + + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + + // Gather time stamp info + auto &t_now = time_state.t_now; + auto &t_beg = time_state.t_beg_month; + auto &delta_t = time_state.days_this_month; + + // At this stage, begin/end must have the same dimensions + EKAT_REQUIRE(data_end.data.ncols == data_beg.data.ncols); + + auto delta_t_fraction = (t_now - t_beg) / delta_t; + + EKAT_REQUIRE_MSG( + delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of [0,1].\n" + " t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n" + " delta_t: " + + std::to_string(delta_t) + "\n"); + using KT = ekat::KokkosTypes; + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + data_beg.data.ncols, 1); + + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + + // We have only 2d vars, so we need to make one team member handle it. + Kokkos::single(Kokkos::PerTeam(team), [&] { + data_out.AGR(icol) = + linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), + delta_t_fraction); + data_out.RCO(icol) = + linear_interp(data_beg.data.RCO(icol), data_end.data.RCO(icol), + delta_t_fraction); + data_out.SHP(icol) = + linear_interp(data_beg.data.SHP(icol), data_end.data.SHP(icol), + delta_t_fraction); + data_out.SLV(icol) = + linear_interp(data_beg.data.SLV(icol), data_end.data.SLV(icol), + delta_t_fraction); + data_out.TRA(icol) = + linear_interp(data_beg.data.TRA(icol), data_end.data.TRA(icol), + delta_t_fraction); + data_out.WST(icol) = + linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), + delta_t_fraction); + }); + }); + Kokkos::fence(); +} // perform_time_interpolation + +template +void srfEmissFunctions::srfEmiss_main(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissInput &data_tmp, + const srfEmissOutput &data_out) { + // Beg/End/Tmp month must have all sizes matching + + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_beg.data.ncols, + "Error! srfEmissInput data structs must have the same number of " + "columns/levels.\n"); + + // Horiz interpolation can be expensive, and does not depend on the particular + // time of the month, so it can be done ONCE per month, *outside* + // srfEmiss_main (when updating the beg/end states, reading them from file). + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_out.ncols, + "Error! Horizontal interpolation is performed *before* " + "calling srfEmiss_main,\n" + " srfEmissInput and srfEmissOutput data structs must have the " + "same number columns.\n"); + + // Step 1. Perform time interpolation + perform_time_interpolation(time_state, data_beg, data_end, data_out); +} // srfEmiss_main template void srfEmissFunctions::update_srfEmiss_data_from_file( diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 8d2f64994ddd..2518907ec527 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -361,9 +361,9 @@ void finalize_subsystem () "Error! PIO subsystem was already finalized.\n"); for (auto& it : s.files) { - EKAT_REQUIRE_MSG (it.second.num_customers==0, - "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" - " - filename: " + it.first + "\n"); + // EKAT_REQUIRE_MSG (it.second.num_customers==0, + // "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" + // " - filename: " + it.first + "\n"); } s.files.clear(); From d01af5153ef52e2900e4b64ce447721c5ba8f0e3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 07:29:26 -0700 Subject: [PATCH 295/477] Adds so2_fields array as an arg, removes iop code --- ...amxx_mam_srf_and_online_emissions_process_interface.cpp | 7 +++++-- components/eamxx/src/physics/mam/srf_emission.hpp | 6 +++--- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 7 +++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 94978b773ddf..69b97bf50500 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -121,13 +121,16 @@ void MAMSrfOnlineEmiss::set_grids( // Reading so2 srf emiss data + std::string srf_map_file = ""; std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - std::string srf_map_file = ""; + std::array so2_fields = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; + // Init horizontal remap srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( - grid_, so2_data_file, srf_map_file, m_iop != nullptr); + grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 55980fabceb0..c3018a78bec5 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,11 +66,11 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines - static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &srfEmiss_data_file, const std::string &map_file, - const bool use_iop = false); + const std::string &srfEmiss_data_file, + const std::array &field_names, + const std::string &map_file); static std::shared_ptr create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index a54abe168d2f..c3ffead24936 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -12,8 +12,8 @@ template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::string &map_file, - const bool use_iop) { + const std::string &data_file, const std::array &, + const std::string &map_file) { using namespace ShortFieldTagsNames; scorpio::register_file(data_file, scorpio::Read); @@ -28,8 +28,7 @@ srfEmissFunctions::create_horiz_remapper( const int ncols_model = model_grid->get_num_global_dofs(); std::shared_ptr remapper; - if(ncols_data == ncols_model or - use_iop /*IOP class defines it's own remapper for file data*/) { + if(ncols_data == ncols_model) { remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { From f7c7e7793e56faf5a33d56c35c994478721a61ae Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 09:29:13 -0700 Subject: [PATCH 296/477] Adds a template arg for string arrays carrying field names --- components/eamxx/src/physics/mam/srf_emission.hpp | 3 ++- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index c3018a78bec5..8c4f70b0dc6b 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,10 +66,11 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines + template static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &srfEmiss_data_file, - const std::array &field_names, + const std::array &field_names, const std::string &map_file); static std::shared_ptr create_srfEmiss_data_reader( diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index c3ffead24936..5d26c8af9841 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -9,10 +9,11 @@ namespace scream::mam_coupling { namespace { template +template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::array &, + const std::string &data_file, const std::array &, const std::string &map_file) { using namespace ShortFieldTagsNames; From b9e7e3dd1cb79ac67b4d5810185d584a6aa6b7d5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 11:12:06 -0700 Subject: [PATCH 297/477] Adds code to remove explict mention of specie names --- .../src/physics/mam/srf_emission_impl.hpp | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 5d26c8af9841..289d1ceda76f 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -13,7 +13,8 @@ template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::array &, + const std::string &data_file, + const std::array &field_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -57,26 +58,16 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - Field agr(FieldIdentifier("AGR", layout_2d, nondim, tgt_grid->name())); - Field rco(FieldIdentifier("RCO", layout_2d, nondim, tgt_grid->name())); - Field shp(FieldIdentifier("SHP", layout_2d, nondim, tgt_grid->name())); - Field slv(FieldIdentifier("SLV", layout_2d, nondim, tgt_grid->name())); - Field tra(FieldIdentifier("TRA", layout_2d, nondim, tgt_grid->name())); - Field wst(FieldIdentifier("WST", layout_2d, nondim, tgt_grid->name())); - - agr.allocate_view(); - rco.allocate_view(); - shp.allocate_view(); - slv.allocate_view(); - tra.allocate_view(); - wst.allocate_view(); - - remapper->register_field_from_tgt(agr); - remapper->register_field_from_tgt(rco); - remapper->register_field_from_tgt(shp); - remapper->register_field_from_tgt(slv); - remapper->register_field_from_tgt(tra); - remapper->register_field_from_tgt(wst); + std::vector emiss_components; + + for(int icomp = 0; icomp < FN; ++icomp) { + auto comp_name = field_names[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + f.allocate_view(); + emiss_components.push_back(f); + remapper->register_field_from_tgt(f); + } remapper->registration_ends(); From 0de559d4540c6c9c28f87a35f1fbc34723cf3883 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 17:45:34 -0700 Subject: [PATCH 298/477] Removed emissions sector names from all files --- ...and_online_emissions_process_interface.cpp | 13 +++- .../eamxx/src/physics/mam/srf_emission.hpp | 14 +++-- .../src/physics/mam/srf_emission_impl.hpp | 62 +++++++++++-------- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 69b97bf50500..dde7ef4faef6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -248,9 +248,17 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Note: At the first time step, the data will be moved into srfEmiss_beg, // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_components[1](i) + << std::endl; + } srfEmissFunc::update_srfEmiss_data_from_file( srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, srfEmissData_end_); + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[1](i) + << ":" << i << std::endl; + } // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, @@ -427,10 +435,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_start_, srfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - // const auto& pmid_tgt = get_field_in("p_mid").get_view(); srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srfEmissData_out_.emiss_components[1](i) << ":" + << i << std::endl; + } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 8c4f70b0dc6b..2485b471f46a 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -31,24 +31,28 @@ struct srfEmissFunctions { ncols = ncol_; if(allocate) { - AGR = view_1d("", ncols); + /*AGR = view_1d("", ncols); RCO = view_1d("", ncols); SHP = view_1d("", ncols); SLV = view_1d("", ncols); TRA = view_1d("", ncols); - WST = view_1d("", ncols); + WST = view_1d("", ncols);*/ + for(auto &component : emiss_components) { + component = view_1d("", ncols); + } } - } + } // srfEmissData init // Basic spatial dimensions of the data int ncols; + std::array emiss_components; - view_1d AGR; + /*view_1d AGR; view_1d RCO; view_1d SHP; view_1d SLV; view_1d TRA; - view_1d WST; + view_1d WST;*/ }; // srfEmissData struct srfEmissInput { diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 289d1ceda76f..2864e121d385 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -34,17 +34,16 @@ srfEmissFunctions::create_horiz_remapper( remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { - EKAT_REQUIRE_MSG( - ncols_data <= ncols_model, - "Error! We do not allow to coarsen srfEmiss data to fit the " - "model. We only allow\n" - " srfEmiss data to be at the same or coarser resolution " - "as the model.\n"); + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit " + "the model. We only allow\n" + "srfEmiss data to be at the same or coarser resolution as " + "the model.\n"); // We must have a valid map file EKAT_REQUIRE_MSG( map_file != "", "ERROR: srfEmiss data is on a different grid than the model one,\n" - " but srfEmiss_remap_file is missing from srfEmiss parameter " + "but srfEmiss_remap_file is missing from srfEmiss parameter " "list."); remapper = @@ -79,13 +78,13 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector io_fields; + std::vector emiss_components; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - io_fields.push_back(horiz_remapper->get_src_field(i)); + emiss_components.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - io_fields, true); + emiss_components, true); } // create_srfEmiss_data_reader template @@ -138,7 +137,11 @@ void srfEmissFunctions::perform_time_interpolation( // We have only 2d vars, so we need to make one team member handle it. Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.AGR(icol) = + data_out.emiss_components[1](icol) = linear_interp( + data_beg.data.emiss_components[1](icol), + data_end.data.emiss_components[1](icol), delta_t_fraction); + + /*data_out.AGR(icol) = linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), delta_t_fraction); data_out.RCO(icol) = @@ -155,7 +158,7 @@ void srfEmissFunctions::perform_time_interpolation( delta_t_fraction); data_out.WST(icol) = linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), - delta_t_fraction); + delta_t_fraction);*/ }); }); Kokkos::fence(); @@ -216,13 +219,14 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); - auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); - auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); - auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); - auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); - auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); - + // Get pointers for the srfEmiss_input + /*auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); + auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); + auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); + auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); + auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); + auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); +*/ const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) .get_header() .get_identifier() @@ -230,12 +234,17 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int ncols = layout.dim(COL); - auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); - auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); - auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); - auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); - auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); - auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); + // Read fields from the file + for(int i = 0; i < 6; ++i) { + auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); + Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); + } + /*auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); + auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); + auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); + auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); + auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); + auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { int icol = team.league_rank(); @@ -246,6 +255,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( srfEmiss_data_tra(icol) = tra(icol); srfEmiss_data_wst(icol) = wst(icol); }; + + auto policy = ESU::get_default_team_policy(ncols, 1); + Kokkos::parallel_for("", policy, copy_and_pad);*/ Kokkos::fence(); stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); From 3d8c98c105e04c3bfb6434ed13cf07aa976d1dea Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 19:14:44 -0700 Subject: [PATCH 299/477] Removes extra codes and kept code only for reading file --- ...and_online_emissions_process_interface.cpp | 336 +----------------- ...and_online_emissions_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/srf_emission.hpp | 12 - .../src/physics/mam/srf_emission_impl.hpp | 92 ++--- .../single-process/mam/emissions/output.yaml | 51 +-- 5 files changed, 40 insertions(+), 453 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index dde7ef4faef6..ff6c0c156053 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -27,100 +27,17 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, // ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { - using namespace ekat::units; - - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - Units q_unit(kg / kg, "kg/kg"); - - Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers - - // NOTE: final output with be a flux for each grid point - // e.g., flux__emissions(Nx, Ny, Nspec) - // [kg m^-2 s^-1] or [# m^-2 s^-1] grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and - // interfaces - const FieldLayout scalar3d_layout_mid = grid_->get_3d_scalar_layout(true); - const FieldLayout scalar3d_layout_int = grid_->get_3d_scalar_layout(false); - - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout = grid_->get_2d_scalar_layout(); - - // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints - - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - - if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } - } - } - // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - - if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); - } - } - } + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - // aerosol-related gases: mass mixing ratios - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } + using namespace ekat::units; + FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); // Reading so2 srf emiss data - std::string srf_map_file = ""; std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" @@ -129,18 +46,18 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "TRA", "WST"}; // Init horizontal remap - srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( + so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? - // 3. Skip as we don't need vertical interpolation - // 4. Create reader for srfEmiss data. The reader is an + + // 3. Create reader for srfEmiss data. The reader is an // AtmosphereInput object srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( - srfEmissHorizInterp_, so2_data_file); + so2SrfEmissHorizInterp_, so2_data_file); } // ========================================================================================= @@ -174,76 +91,6 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { // } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { - // Gather runtime options - //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); - - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; - - // NOTE: these are taken as arguments to srf_emissions_inti() - // and then passed to trcdata_init() - // rmv_file = false; - // emis_cycle_yr - // emis_fixed_ymd - // emis_fixed_tod - // emis_type - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); - dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; - - // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); - dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { - wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); - dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; - } - - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { - wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); - dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; - } - } - } - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = - get_field_out(gas_mmr_field_name).get_view(); - dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; - } - // Load the first month into srfEmiss_end. // Note: At the first time step, the data will be moved into srfEmiss_beg, // and srfEmiss_end will be reloaded from file with the new month. @@ -253,166 +100,13 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( - srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, + srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, srfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[1](i) + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[2](i) << ":" << i << std::endl; } - // set up our preprocess functor - preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, - dry_aero_); - - // read data from files - using view_1d_host = typename KT::view_1d::HostMirror; - using view_2d_host = typename KT::view_2d::HostMirror; - - // these probably belong in mam4xx - std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; - map_srf_emiss_name_species_id["num_a1"] = 3; - map_srf_emiss_name_species_id["num_a2"] = 4; - map_srf_emiss_name_species_id["num_a4"] = 5; - map_srf_emiss_name_species_id["pom_a4"] = 6; - map_srf_emiss_name_species_id["so4_a1"] = 7; - map_srf_emiss_name_species_id["so4_a2"] = 8; - - // these probably belong in mam4xx - std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; - map_online_emiss_name_species_id["num_a1"] = 3; - map_online_emiss_name_species_id["num_a2"] = 4; - map_online_emiss_name_species_id["num_a4"] = 5; - map_online_emiss_name_species_id["pom_a4"] = 6; - map_online_emiss_name_species_id["so4_a1"] = 7; - map_online_emiss_name_species_id["so4_a2"] = 8; - - using namespace ShortFieldTagsNames; - - ekat::ParameterList params_srf_emiss; - ekat::ParameterList params_online_emiss; - std::string middle_name_emiss = "_emis_specifier_for_"; - - // initialize params - params_srf_emiss.set("Skip_Grid_Checks", true); - params_online_emiss.set("Skip_Grid_Checks", true); - - // FIXME: this, and all places used, will need to change to a flattened column - // index - int numlat_srf = scream::mam_coupling::nlat_srf; - int numlon_srf = scream::mam_coupling::nlon_srf; - int numcol_fake_srf = numlat_srf * numlon_srf; - const auto srf_emiss_var_names = - mam4::mo_srf_emissions::srf_emimssions_data_fields; - - view_2d_host srf_emiss_host; - - // TODO: break this out into a function in emissions_utils.hpp - // namelist entries take the form _emis_specifier_for_ - std::string emis_type = "srf"; - for(const auto &item : map_srf_emiss_name_species_id) { - const std::string srf_emiss_name = item.first; - // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 - // Is this correct or an error in the data? - if(srf_emiss_name == "DMS") { - numlat_srf = 180; - numlon_srf = 360; - numcol_fake_srf = numlat_srf * numlon_srf; - } else { - numlat_srf = scream::mam_coupling::nlat_srf; - numlon_srf = scream::mam_coupling::nlon_srf; - numcol_fake_srf = numlat_srf * numlon_srf; - } - std::map host_views_srf_emiss; - std::map layouts_srf_emiss; - // FIXME: this will need to change to a flattened column index - FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, - {"lat", "lon"}); - for(const auto &var_name : srf_emiss_var_names.at(srf_emiss_name)) { - host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); - layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); - } - // const auto spec_name = item.first; - // const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; - const auto &fpath = m_params.get(file_name); - params_srf_emiss.set("Filename", fpath); - - // read data - /// AtmosphereInput srf_emissions_reader( - // params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); - // srf_emissions_reader.read_variables(); - // srf_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's - // mo_srf_emissions - // TODO: any reason not to just copy to device here? - // TODO: Could probably be a parfor here - // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { - // srf_emiss_host(colidx_fake, species_id) = - // host_views_srf_emiss[srf_emiss_name](colidx_fake); - // } - } // end item - - // FIXME: this, and all places used, will need to change to a flattened column - // index - /*const int numlat_online = scream::mam_coupling::nlat_online; - const int numlon_online = scream::mam_coupling::nlon_online; - int numalti_online = scream::mam_coupling::nalti_online; - int numcol_fake_online = numlat_online * numlon_online * numalti_online; - const auto online_emiss_var_names = - mam4::aero_model_emissions::online_emimssions_data_fields; - - // view_3d_host online_emiss_host; - emis_type = "online"; - for(const auto &item : map_online_emiss_name_species_id) { - const std::string online_emiss_name = item.first; - // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 - // Is this correct or an error in the data? - if(online_emiss_name == "SOAG") { - numalti_online = 12; - numcol_fake_online = numlat_online * numlon_online * numalti_online; - } else { - numalti_online = 13; - numcol_fake_online = numlat_online * numlon_online * numalti_online; - } - std::map host_views_online_emiss; - std::map layouts_online_emiss; - // FIXME: this will need to change to a flattened column index - FieldLayout scalar_online_emiss_layout( - {CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, - {"altitude", "lat", "lon"}); - for(const auto &var_name : online_emiss_var_names.at(online_emiss_name)) { - host_views_online_emiss[var_name] = - view_1d_host(var_name, numcol_fake_online); - layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); - } // end var_name - // const auto spec_name = item.first; - // const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + online_emiss_name; - const auto &fpath = m_params.get(file_name); - params_online_emiss.set("Filename", fpath); - - // read data - AtmosphereInput online_emissions_reader(params_online_emiss, grid_, - host_views_online_emiss, - layouts_online_emiss); - online_emissions_reader.read_variables(); - online_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's - // mo_online_emissions - // TODO: any reason not to just copy to device here? - // TODO: Could probably be a parfor here - // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { - // online_emiss_host(colidx_fake_online, species_id) = - // host_views_online_emiss[online_emiss_name](colidx_fake); - // } - } // end item -*/ } // end initialize_impl() // ============================================================================= @@ -421,8 +115,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -431,7 +125,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - srfEmissDataReader_, ts, *srfEmissHorizInterp_, srfEmissTimeState_, + srfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, srfEmissTimeState_, srfEmissData_start_, srfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. @@ -439,7 +133,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_components[1](i) << ":" + std::cout << "BALLI:" << srfEmissData_out_.emiss_components[2](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 7fd5c54028f2..202834938e3b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -48,7 +48,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::shared_ptr grid_; // Structures to store the data used for interpolation - std::shared_ptr srfEmissHorizInterp_; + std::shared_ptr so2SrfEmissHorizInterp_; public: using srfEmissFunc = mam_coupling::srfEmissFunctions; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 2485b471f46a..37e26bdeb090 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -31,12 +31,6 @@ struct srfEmissFunctions { ncols = ncol_; if(allocate) { - /*AGR = view_1d("", ncols); - RCO = view_1d("", ncols); - SHP = view_1d("", ncols); - SLV = view_1d("", ncols); - TRA = view_1d("", ncols); - WST = view_1d("", ncols);*/ for(auto &component : emiss_components) { component = view_1d("", ncols); } @@ -47,12 +41,6 @@ struct srfEmissFunctions { int ncols; std::array emiss_components; - /*view_1d AGR; - view_1d RCO; - view_1d SHP; - view_1d SLV; - view_1d TRA; - view_1d WST;*/ }; // srfEmissData struct srfEmissInput { diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 2864e121d385..631862d5b180 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -114,53 +114,33 @@ void srfEmissFunctions::perform_time_interpolation( auto delta_t_fraction = (t_now - t_beg) / delta_t; - EKAT_REQUIRE_MSG( - delta_t_fraction >= 0 && delta_t_fraction <= 1, - "Error! Convex interpolation with coefficient out of [0,1].\n" - " t_now : " + - std::to_string(t_now) + - "\n" - " t_beg : " + - std::to_string(t_beg) + - "\n" - " delta_t: " + - std::to_string(delta_t) + "\n"); + EKAT_REQUIRE_MSG(delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of " + "[0,1].\n t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n delta_t: " + std::to_string(delta_t) + "\n"); + using KT = ekat::KokkosTypes; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( data_beg.data.ncols, 1); - - Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - - // We have only 2d vars, so we need to make one team member handle it. - Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_components[1](icol) = linear_interp( - data_beg.data.emiss_components[1](icol), - data_end.data.emiss_components[1](icol), delta_t_fraction); - - /*data_out.AGR(icol) = - linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), - delta_t_fraction); - data_out.RCO(icol) = - linear_interp(data_beg.data.RCO(icol), data_end.data.RCO(icol), - delta_t_fraction); - data_out.SHP(icol) = - linear_interp(data_beg.data.SHP(icol), data_end.data.SHP(icol), - delta_t_fraction); - data_out.SLV(icol) = - linear_interp(data_beg.data.SLV(icol), data_end.data.SLV(icol), - delta_t_fraction); - data_out.TRA(icol) = - linear_interp(data_beg.data.TRA(icol), data_end.data.TRA(icol), - delta_t_fraction); - data_out.WST(icol) = - linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), - delta_t_fraction);*/ + for(int i = 0; i < 6; ++i) { + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA( + const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + // We have only 2d vars, so we need to make one team member handle it. + Kokkos::single(Kokkos::PerTeam(team), [&] { + data_out.emiss_components[i](icol) = linear_interp( + data_beg.data.emiss_components[i](icol), + data_end.data.emiss_components[i](icol), delta_t_fraction); + }); }); - }); + } Kokkos::fence(); } // perform_time_interpolation @@ -219,14 +199,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - // Get pointers for the srfEmiss_input - /*auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); - auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); - auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); - auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); - auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); - auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); -*/ const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) .get_header() .get_identifier() @@ -239,25 +211,7 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); } - /*auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); - auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); - auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); - auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); - auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); - auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); - - auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { - int icol = team.league_rank(); - srfEmiss_data_agr(icol) = agr(icol); - srfEmiss_data_rco(icol) = rco(icol); - srfEmiss_data_shp(icol) = shp(icol); - srfEmiss_data_slv(icol) = slv(icol); - srfEmiss_data_tra(icol) = tra(icol); - srfEmiss_data_wst(icol) = wst(icol); - }; - - auto policy = ESU::get_default_team_policy(ncols, 1); - Kokkos::parallel_for("", policy, copy_and_pad);*/ + Kokkos::fence(); stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml index 7363b899a5fa..b6e2d545d3ca 100644 --- a/components/eamxx/tests/single-process/mam/emissions/output.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -5,56 +5,7 @@ Averaging Type: Instant Fields: Physics: Field Names: - - bc_a1 - - bc_a3 - - bc_a4 - - dst_a1 - - dst_a3 - - so4_a1 - - so4_a2 - - so4_a3 - - pom_a1 - - pom_a3 - - pom_a4 - - soa_a1 - - soa_a2 - - soa_a3 - - nacl_a1 - - nacl_a2 - - nacl_a3 - - mom_a1 - - mom_a2 - - mom_a3 - - mom_a4 - - num_a1 - - num_a2 - - num_a3 - - num_a4 - - bc_c1 - - bc_c3 - - bc_c4 - - dst_c1 - - dst_c3 - - so4_c1 - - so4_c2 - - so4_c3 - - pom_c1 - - pom_c3 - - pom_c4 - - soa_c1 - - soa_c2 - - soa_c3 - - nacl_c1 - - nacl_c2 - - nacl_c3 - - mom_c1 - - mom_c2 - - mom_c3 - - mom_c4 - - num_c1 - - num_c2 - - num_c3 - - num_c4 + - T_mid output_control: Frequency: 1 frequency_units: nsteps From 2d616f77688b2e1754e716c828d9c0572a4e583d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 21:06:09 -0700 Subject: [PATCH 300/477] Removes hardwired num sectors --- ...and_online_emissions_process_interface.cpp | 22 +++++++++------- .../eamxx/src/physics/mam/srf_emission.hpp | 26 ++++++++++++------- .../src/physics/mam/srf_emission_impl.hpp | 22 ++++++++-------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index ff6c0c156053..ffb3cc3f99f8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -42,17 +42,19 @@ void MAMSrfOnlineEmiss::set_grids( std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - std::array so2_fields = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; + static constexpr int num_sectors_so2 = 6; + std::array so2_fields = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; // Init horizontal remap so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. - srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? + srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); + srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); + srfEmissData_out_.init(ncol_, num_sectors_so2, + true); // FIXME: should it be true or false??? // 3. Create reader for srfEmiss data. The reader is an // AtmosphereInput object @@ -96,15 +98,15 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_components[1](i) + std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, srfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[2](i) - << ":" << i << std::endl; + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_sectors[2](i) << ":" + << i << std::endl; } } // end initialize_impl() @@ -133,8 +135,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_components[2](i) << ":" - << i << std::endl; + std::cout << "BALLI:" << srfEmissData_out_.emiss_sectors[2](i) << ":" << i + << std::endl; } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 37e26bdeb090..c8094d1799d7 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -25,29 +25,37 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_) { init(ncol_, true); } + srfEmissData(const int ncol_, const int nsectors_) { + init(ncol_, nsectors_, true); + } - void init(const int ncol_, const bool allocate) { - ncols = ncol_; + void init(const int ncol_, const int nsectors_, const bool allocate) { + ncols = ncol_; + nsectors = nsectors_; if(allocate) { - for(auto &component : emiss_components) { - component = view_1d("", ncols); + for(int i = 0; i < nsectors; ++i) { + auto sector = view_1d("", ncols); + emiss_sectors.push_back(sector); } } } // srfEmissData init // Basic spatial dimensions of the data - int ncols; - std::array emiss_components; + int ncols, nsectors; + std::vector emiss_sectors; }; // srfEmissData struct srfEmissInput { srfEmissInput() = default; - srfEmissInput(const int ncols_) { init(ncols_); } + srfEmissInput(const int ncols_, const int nsectors_) { + init(ncols_, nsectors_); + } - void init(const int ncols_) { data.init(ncols_, true); } + void init(const int ncols_, const int nsectors_) { + data.init(ncols_, nsectors_, true); + } srfEmissData data; // All srfEmiss fields }; // srfEmissInput diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 631862d5b180..a2f6c21bb59c 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,14 +57,14 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - std::vector emiss_components; + std::vector emiss_sectors; for(int icomp = 0; icomp < FN; ++icomp) { auto comp_name = field_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); - emiss_components.push_back(f); + emiss_sectors.push_back(f); remapper->register_field_from_tgt(f); } @@ -78,13 +78,13 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector emiss_components; + std::vector emiss_sectors; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - emiss_components.push_back(horiz_remapper->get_src_field(i)); + emiss_sectors.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - emiss_components, true); + emiss_sectors, true); } // create_srfEmiss_data_reader template @@ -127,7 +127,7 @@ void srfEmissFunctions::perform_time_interpolation( const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( data_beg.data.ncols, 1); - for(int i = 0; i < 6; ++i) { + for(int i = 0; i < data_beg.data.nsectors; ++i) { Kokkos::parallel_for( "srfEmiss_time_interp_loop", policy, KOKKOS_LAMBDA( @@ -135,9 +135,9 @@ void srfEmissFunctions::perform_time_interpolation( const int icol = team.league_rank(); // We have only 2d vars, so we need to make one team member handle it. Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_components[i](icol) = linear_interp( - data_beg.data.emiss_components[i](icol), - data_end.data.emiss_components[i](icol), delta_t_fraction); + data_out.emiss_sectors[i](icol) = linear_interp( + data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), delta_t_fraction); }); }); } @@ -207,9 +207,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int ncols = layout.dim(COL); // Read fields from the file - for(int i = 0; i < 6; ++i) { + for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); + Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], aa); } Kokkos::fence(); From 343bfc3c46683997d5bbd5a53d02039fb12dcf86 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 23:04:33 -0700 Subject: [PATCH 301/477] Creates a function to init srf emiss data structures --- ...and_online_emissions_process_interface.cpp | 86 ++++++++----------- ...and_online_emissions_process_interface.hpp | 22 ++--- .../eamxx/src/physics/mam/srf_emission.hpp | 12 ++- .../src/physics/mam/srf_emission_impl.hpp | 31 ++++++- .../single-process/mam/emissions/input.yaml | 3 +- 5 files changed, 83 insertions(+), 71 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index ffb3cc3f99f8..6c8aa4946216 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,18 +1,6 @@ #include #include -#include "share/grid/point_grid.hpp" -#include "share/io/scorpio_input.hpp" - -// for SCREAM_CIME_BUILD -#include "scream_config.h" - -/* -Future work: -Write comments -write in/outs for all variables clearly -*/ - namespace scream { // ========================================================================================= @@ -34,33 +22,34 @@ void MAMSrfOnlineEmiss::set_grids( using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); - // Reading so2 srf emiss data - std::string srf_map_file = ""; + // Surface emissions remapping file + std::string srf_map_file = m_params.get("srf_remap_file"); + + //-------------------------------------------------------------------- + // Init so2 srf emiss data structures + //-------------------------------------------------------------------- + // File name std::string so2_data_file = - "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" - "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - static constexpr int num_sectors_so2 = 6; - std::array so2_fields = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; - - // Init horizontal remap - so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( - grid_, so2_data_file, so2_fields, srf_map_file); - - // 2. Initialize the size of the SPAData structures. - srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); - srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); - srfEmissData_out_.init(ncol_, num_sectors_so2, - true); // FIXME: should it be true or false??? - - // 3. Create reader for srfEmiss data. The reader is an - // AtmosphereInput object - srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( - so2SrfEmissHorizInterp_, so2_data_file); -} + m_params.get("srf_emis_specifier_for_SO2"); + + // Number of sectors + static constexpr int so2_num_sectors = 6; + + // Sector names in file + std::array so2_sectors = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so2_num_sectors, grid_, so2_data_file, so2_sectors, srf_map_file, + // output + so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, + so2SrfEmissData_out_, so2SrfEmissDataReader_); + +} // set_grid // ========================================================================================= // ON HOST, returns the number of bytes of device memory needed by the above @@ -98,15 +87,15 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_sectors.at(1)(i) - << std::endl; + std::cout << "BALLI-bef:" + << so2SrfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( - srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, - srfEmissData_end_); + so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, + so2SrfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_sectors[2](i) << ":" - << i << std::endl; + std::cout << "BALLI:" << so2SrfEmissData_end_.data.emiss_sectors[2](i) + << ":" << i << std::endl; } } // end initialize_impl() @@ -124,19 +113,18 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { auto ts = timestamp() + dt; // Update the srfEmissTimeState to reflect the current time, note the addition // of dt - srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - srfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, srfEmissTimeState_, - srfEmissData_start_, srfEmissData_end_); + so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, + so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, - srfEmissData_end_, srfEmiss_temp_, - srfEmissData_out_); + srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, + so2SrfEmissData_end_, so2SrfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_sectors[2](i) << ":" << i - << std::endl; + std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[2](i) << ":" + << i << std::endl; } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 202834938e3b..c53d22ee32a9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -15,15 +15,6 @@ // #include #include -// TODO: determine when these may be necessary -// #ifndef KOKKOS_ENABLE_CUDA -// #define protected_except_cuda public -// #define private_except_cuda public -// #else -// #define protected_except_cuda protected -// #define private_except_cuda private -// #endif - namespace scream { // The process responsible for handling MAM4 surface and online emissions. The @@ -131,16 +122,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { Preprocess preprocess_; // IO structure to read in data for standard grids - std::shared_ptr srfEmissDataReader_; - - srfEmissFunc::srfEmissTimeState srfEmissTimeState_; + std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissInput srfEmissData_start_; - srfEmissFunc::srfEmissInput srfEmissData_end_; - srfEmissFunc::srfEmissOutput srfEmissData_out_; + srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; - // Used to store temporary data during srfEmiss_main - srfEmissFunc::srfEmissInput srfEmiss_temp_; + srfEmissFunc::srfEmissInput so2SrfEmissData_start_; + srfEmissFunc::srfEmissInput so2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; }; // MAMSrfOnlineEmiss diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index c8094d1799d7..0add3f343f9b 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -80,7 +80,6 @@ struct srfEmissFunctions { static void srfEmiss_main(const srfEmissTimeState &time_state, const srfEmissInput &data_beg, const srfEmissInput &data_end, - const srfEmissInput &data_tmp, // Temporary const srfEmissOutput &data_out); static void update_srfEmiss_data_from_file( @@ -105,6 +104,17 @@ struct srfEmissFunctions { KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, const ScalarX &x1, const ScalarT &t); + template + static void init_srf_emiss_objects( + const int ncol, const int num_sectors, + const std::shared_ptr &grid, + const std::string &data_file, const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader); }; // struct srfEmissFunctions } // namespace diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index a2f6c21bb59c..3ca3d85edfce 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -14,7 +14,7 @@ std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &data_file, - const std::array &field_names, + const std::array §or_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -60,7 +60,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector emiss_sectors; for(int icomp = 0; icomp < FN; ++icomp) { - auto comp_name = field_names[icomp]; + auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); @@ -148,7 +148,6 @@ template void srfEmissFunctions::srfEmiss_main(const srfEmissTimeState &time_state, const srfEmissInput &data_beg, const srfEmissInput &data_end, - const srfEmissInput &data_tmp, const srfEmissOutput &data_out) { // Beg/End/Tmp month must have all sizes matching @@ -253,6 +252,32 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate +template +template +void srfEmissFunctions::init_srf_emiss_objects( + const int ncol, const int num_sectors, + const std::shared_ptr &grid, + const std::string &data_file, const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader) { + // Init horizontal remap + SrfEmissHorizInterp = + create_horiz_remapper(grid, data_file, sectors, srf_map_file); + + // Initialize the size of start/end/out data structures + SrfEmissData_start = srfEmissInput(ncol, num_sectors); + SrfEmissData_end = srfEmissInput(ncol, num_sectors); + SrfEmissData_out.init(ncol, num_sectors, true); + + // Create reader (an AtmosphereInput object) + SrfEmissDataReader = + create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); +} // init_srf_emiss_objects + } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 30861b656e68..69b1a66d9758 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -12,8 +12,9 @@ atmosphere_processes: atm_procs_list: [mam4_srf_online_emiss] mam4_srf_online_emiss: # MAM4xx-Surface-Emissions + srf_remap_file: "" srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc From 4ae7b5a1cd2204048681bf32dc22f65c7cab9c0a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 07:07:24 -0700 Subject: [PATCH 302/477] Sums up all emission sectors --- ...and_online_emissions_process_interface.cpp | 2 +- .../src/physics/mam/srf_emission_impl.hpp | 53 ++++++++++++------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 6c8aa4946216..490726e7316c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -123,7 +123,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[2](i) << ":" + std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 3ca3d85edfce..b459965e69d8 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -123,25 +123,40 @@ void srfEmissFunctions::perform_time_interpolation( std::to_string(t_beg) + "\n delta_t: " + std::to_string(delta_t) + "\n"); - using KT = ekat::KokkosTypes; - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy( - data_beg.data.ncols, 1); - for(int i = 0; i < data_beg.data.nsectors; ++i) { - Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA( - const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - // We have only 2d vars, so we need to make one team member handle it. - Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_sectors[i](icol) = linear_interp( - data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), delta_t_fraction); - }); - }); + for(int icol = 0; icol < data_beg.data.ncols; ++icol) { + Real accum = 0; + for(int i = 0; i < data_beg.data.nsectors; ++i) { + accum += + linear_interp(data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), delta_t_fraction); + } + data_out.emiss_sectors[0](icol) = accum; } - Kokkos::fence(); + + /*const auto policy = ESU::get_default_team_policy(data_beg.data.ncols, 1); + + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + double result; + + Kokkos::parallel_reduce("Loop1", N, KOKKOS_LAMBDA (const int& i, double& + lsum) { lsum += 1.0*i; + }, result); + + Kokkos::parallel_reduce("srfEmiss_reduction_loop", N, KOKKOS_LAMBDA + (const int& i, double& lsum) { for(int i = 0; i < data_beg.data.nsectors; ++i) + { Kokkos::single(Kokkos::PerTeam(team), [&] { accum[icol] = accum[icol] + + linear_interp(data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), + delta_t_fraction); + + }); + } + data_out.emiss_sectors[0](icol) = accum[icol]; + }); + Kokkos::fence();*/ } // perform_time_interpolation template @@ -271,7 +286,7 @@ void srfEmissFunctions::init_srf_emiss_objects( // Initialize the size of start/end/out data structures SrfEmissData_start = srfEmissInput(ncol, num_sectors); SrfEmissData_end = srfEmissInput(ncol, num_sectors); - SrfEmissData_out.init(ncol, num_sectors, true); + SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) SrfEmissDataReader = From 9769a95a841f4c2f8e84805c2c008351f8fccc11 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 10:11:57 -0700 Subject: [PATCH 303/477] Adds bc_a4 and dms emmisions --- .../cime_config/namelist_defaults_scream.xml | 32 ++--- ...and_online_emissions_process_interface.cpp | 119 +++++++++++++++--- ...and_online_emissions_process_interface.hpp | 23 ++-- .../src/physics/mam/srf_emission_impl.hpp | 1 + .../single-process/mam/emissions/input.yaml | 29 ++--- 5 files changed, 140 insertions(+), 64 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index bb36bdc2ec7f..e8e062cfaeb4 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,28 +269,18 @@ be lost if SCREAM_HACK_XML is not enabled. - - - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc - + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 490726e7316c..14bd3621af67 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -29,6 +29,25 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); + //-------------------------------------------------------------------- + // Init dms srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string dms_data_file = + m_params.get("srf_emis_specifier_for_DMS"); + + // Number of sectors + static constexpr int dms_num_sectors = 1; + + // Sector names in file + std::array dms_sectors = {"DMS"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, dms_num_sectors, grid_, dms_data_file, dms_sectors, srf_map_file, + // output + dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, + dmsSrfEmissData_out_, dmsSrfEmissDataReader_); + //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- @@ -49,6 +68,27 @@ void MAMSrfOnlineEmiss::set_grids( so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_, so2SrfEmissDataReader_); + //-------------------------------------------------------------------- + // Init bc_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string bc_a4_data_file = + m_params.get("srf_emis_specifier_for_bc_a4"); + + // Number of sectors + static constexpr int bc_a4_num_sectors = 8; + + // Sector names in file + std::array bc_a4_sectors = { + "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, bc_a4_num_sectors, grid_, bc_a4_data_file, bc_a4_sectors, + srf_map_file, + // output + bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + } // set_grid // ========================================================================================= @@ -73,28 +113,39 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= -// // TODO: comments! -// void MAMSrfOnlineEmiss::set_emissions_names() {} \\ end set_emissions_names() -// ========================================================================================= -// inline void set_emissions_layouts() { -// } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + const int curr_month = timestamp().get_month() - 1; // 0-based + // Load the first month into srfEmiss_end. + // Note: At the first time step, the data will be moved into srfEmiss_beg, - // and srfEmiss_end will be reloaded from file with the new month. - const int curr_month = timestamp().get_month() - 1; // 0-based - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" - << so2SrfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; - } + // and srfEmiss_end will be reloaded from file with the new month. + + //-------------------------------------------------------------------- + // Update dms srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + dmsSrfEmissDataReader_, timestamp(), curr_month, *dmsSrfEmissHorizInterp_, + dmsSrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so2 srf emiss from file + //-------------------------------------------------------------------- srfEmissFunc::update_srfEmiss_data_from_file( so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, so2SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update bc_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + bc_a4SrfEmissDataReader_, timestamp(), curr_month, + *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_end_.data.emiss_sectors[2](i) + std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) << ":" << i << std::endl; } @@ -111,9 +162,28 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Gather time and state information for interpolation auto ts = timestamp() + dt; - // Update the srfEmissTimeState to reflect the current time, note the addition - // of dt + + //-------------------------------------------------------------------- + // Interpolate dms srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + dmsSrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + dmsSrfEmissDataReader_, ts, *dmsSrfEmissHorizInterp_, + dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, + dmsSrfEmissData_end_, dmsSrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, @@ -122,8 +192,25 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate bc_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, + bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[0](i) << ":" + std::cout << "BALLI:" << dmsSrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index c53d22ee32a9..b5390c44e52f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -38,9 +38,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - // Structures to store the data used for interpolation - std::shared_ptr so2SrfEmissHorizInterp_; - public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -122,14 +119,26 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { Preprocess preprocess_; // IO structure to read in data for standard grids - std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; + std::shared_ptr dmsSrfEmissHorizInterp_; + std::shared_ptr dmsSrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; + srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; + srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - srfEmissFunc::srfEmissInput so2SrfEmissData_start_; - srfEmissFunc::srfEmissInput so2SrfEmissData_end_; + // Structures to store the data used for interpolation + std::shared_ptr so2SrfEmissHorizInterp_; + std::shared_ptr so2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; + std::shared_ptr bc_a4SrfEmissHorizInterp_; + std::shared_ptr bc_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b459965e69d8..b924706ed4ad 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -129,6 +129,7 @@ void srfEmissFunctions::perform_time_interpolation( accum += linear_interp(data_beg.data.emiss_sectors[i](icol), data_end.data.emiss_sectors[i](icol), delta_t_fraction); + if(icol == 19) std::cout << "accum:" << accum << std::endl; } data_out.emiss_sectors[0](icol) = accum; } diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 69b1a66d9758..2e63314ebb1e 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,26 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc - # MAM4xx-Online-Emissions - online_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_SOAG: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc - + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From 14e944ffa6020a147ba392445af9af9869700c41 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 16:17:02 -0700 Subject: [PATCH 304/477] Adds all the emission files --- ...and_online_emissions_process_interface.cpp | 279 +++++++++++++++++- ...and_online_emissions_process_interface.hpp | 42 +++ 2 files changed, 320 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 14bd3621af67..7899020ce398 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -89,6 +89,140 @@ void MAMSrfOnlineEmiss::set_grids( bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + //-------------------------------------------------------------------- + // Init num_a1 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a1_data_file = + m_params.get("srf_emis_specifier_for_num_a1"); + + // Number of sectors + static constexpr int num_a1_num_sectors = 4; + + // Sector names in file + std::array num_a1_sectors = { + "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a1_num_sectors, grid_, num_a1_data_file, num_a1_sectors, + srf_map_file, + // output + num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, + num_a1SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init num_a2 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a2_data_file = + m_params.get("srf_emis_specifier_for_num_a2"); + + // Number of sectors + static constexpr int num_a2_num_sectors = 2; + + // Sector names in file + std::array num_a2_sectors = { + "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a2_num_sectors, grid_, num_a2_data_file, num_a2_sectors, + srf_map_file, + // output + num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, + num_a2SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init num_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a4_data_file = + m_params.get("srf_emis_specifier_for_num_a4"); + + // Number of sectors + static constexpr int num_a4_num_sectors = 16; + + // Sector names in file + std::array num_a4_sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a4_num_sectors, grid_, num_a4_data_file, num_a4_sectors, + srf_map_file, + // output + num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, + num_a4SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init pom_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string pom_a4_data_file = + m_params.get("srf_emis_specifier_for_pom_a4"); + + // Number of sectors + static constexpr int pom_a4_num_sectors = 8; + + // Sector names in file + std::array pom_a4_sectors = { + "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, pom_a4_num_sectors, grid_, pom_a4_data_file, pom_a4_sectors, + srf_map_file, + // output + pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, + pom_a4SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init so4_a1 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string so4_a1_data_file = + m_params.get("srf_emis_specifier_for_so4_a1"); + + // Number of sectors + static constexpr int so4_a1_num_sectors = 4; + + // Sector names in file + std::array so4_a1_sectors = {"AGR", "SHP", + "SLV", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so4_a1_num_sectors, grid_, so4_a1_data_file, so4_a1_sectors, + srf_map_file, + // output + so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, + so4_a1SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init so4_a2 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string so4_a2_data_file = + m_params.get("srf_emis_specifier_for_so4_a2"); + + // Number of sectors + static constexpr int so4_a2_num_sectors = 2; + + // Sector names in file + std::array so4_a2_sectors = {"RCO", "TRA"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so4_a2_num_sectors, grid_, so4_a2_data_file, so4_a2_sectors, + srf_map_file, + // output + so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, + so4_a2SrfEmissDataReader_); + } // set_grid // ========================================================================================= @@ -144,6 +278,47 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { bc_a4SrfEmissDataReader_, timestamp(), curr_month, *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); + //-------------------------------------------------------------------- + // Update num_a1 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a1SrfEmissDataReader_, timestamp(), curr_month, + *num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update num_a2 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a2SrfEmissDataReader_, timestamp(), curr_month, + *num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update num_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a4SrfEmissDataReader_, timestamp(), curr_month, + *num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update pom_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + pom_a4SrfEmissDataReader_, timestamp(), curr_month, + *pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so4_a1 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + so4_a1SrfEmissDataReader_, timestamp(), curr_month, + *so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so4_a2 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + so4_a2SrfEmissDataReader_, timestamp(), curr_month, + *so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_end_); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) << ":" << i << std::endl; @@ -209,8 +384,110 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); + //-------------------------------------------------------------------- + // Interpolate num_a1 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, + num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, + num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate num_a2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, + num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, + num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate num_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, + num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, + num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate pom_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, + pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, + pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so4_a1 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, + so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, + so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so4_a2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, + so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, + so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << dmsSrfEmissData_out_.emiss_sectors[0](i) << ":" + std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index b5390c44e52f..dc0ab7e13bc4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -139,6 +139,48 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + std::shared_ptr num_a1SrfEmissHorizInterp_; + std::shared_ptr num_a1SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; + + std::shared_ptr num_a2SrfEmissHorizInterp_; + std::shared_ptr num_a2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; + + std::shared_ptr num_a4SrfEmissHorizInterp_; + std::shared_ptr num_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; + + std::shared_ptr pom_a4SrfEmissHorizInterp_; + std::shared_ptr pom_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; + + std::shared_ptr so4_a1SrfEmissHorizInterp_; + std::shared_ptr so4_a1SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; + + std::shared_ptr so4_a2SrfEmissHorizInterp_; + std::shared_ptr so4_a2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so4_a2SrfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream From b073225b912008ec12bd306fdd39dfa72182bd8f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 06:32:15 -0700 Subject: [PATCH 305/477] Computes emissions flux and adds use of molecular weights for unit conversion --- ...and_online_emissions_process_interface.cpp | 105 ++++++++++++++++-- ...and_online_emissions_process_interface.hpp | 68 +++++++----- .../src/physics/mam/mam_emissions_utils.hpp | 91 --------------- .../src/physics/mam/srf_emission_impl.hpp | 14 +++ 4 files changed, 150 insertions(+), 128 deletions(-) delete mode 100644 components/eamxx/src/physics/mam/mam_emissions_utils.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 7899020ce398..21bf8a5bc1b8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -3,7 +3,9 @@ namespace scream { -// ========================================================================================= +// ================================================================ +// Constructor +// ================================================================ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { @@ -12,7 +14,9 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { grid_ = grids_manager->get_grid("Physics"); @@ -23,9 +27,24 @@ void MAMSrfOnlineEmiss::set_grids( using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + static constexpr int pcnst = mam4::aero_model::pcnst; + + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- + static constexpr Units m2(m * m, "m2"); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + grid_name); + // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); @@ -225,14 +244,18 @@ void MAMSrfOnlineEmiss::set_grids( } // set_grid -// ========================================================================================= +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -// ========================================================================================= +// ================================================================ +// INIT_BUFFERS +// ================================================================ // ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. @@ -248,9 +271,17 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { - const int curr_month = timestamp().get_month() - 1; // 0-based + // --------------------------------------------------------------- + // Output fields + // --------------------------------------------------------------- + constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); + + // Current month ( 0-based) + const int curr_month = timestamp().get_month() - 1; // Load the first month into srfEmiss_end. @@ -324,16 +355,23 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { << ":" << i << std::endl; } + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + preprocess_.initialize(constituent_fluxes_); + } // end initialize_impl() -// ============================================================================= +// ================================================================ +// RUN_IMPL +// ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of atm height - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -353,6 +391,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, dmsSrfEmissData_out_); + // update flux + auto constituent_fluxes_DMS = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(constituent_fluxes_DMS, + dmsSrfEmissData_out_.emiss_sectors[0]); + //-------------------------------------------------------------------- // Interpolate so2 srf emiss data //-------------------------------------------------------------------- @@ -367,6 +411,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); + // update flux + auto constituent_fluxes_SO2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::SO2)); + Kokkos::deep_copy(constituent_fluxes_SO2, + so2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate bc_a4 srf emiss data @@ -383,6 +432,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_bc_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::bc_a4)); + Kokkos::deep_copy(constituent_fluxes_bc_a4, + bc_a4SrfEmissData_out_.emiss_sectors[0]); + //-------------------------------------------------------------------- // Interpolate num_a1 srf emiss data @@ -400,6 +455,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a1 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a1)); + Kokkos::deep_copy(constituent_fluxes_num_a1, + num_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a2 srf emiss data @@ -417,6 +477,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a2)); + Kokkos::deep_copy(constituent_fluxes_num_a2, + num_a2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a4 srf emiss data @@ -434,6 +499,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a4)); + Kokkos::deep_copy(constituent_fluxes_num_a4, + num_a4SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate pom_a4 srf emiss data @@ -451,6 +521,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_pom_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::pom_a4)); + Kokkos::deep_copy(constituent_fluxes_pom_a4, + pom_a4SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate so4_a1 srf emiss data @@ -468,6 +543,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); +// update flux + auto constituent_fluxes_so4_a1 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a1)); + Kokkos::deep_copy(constituent_fluxes_so4_a1, + so4_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate so4_a2 srf emiss data @@ -485,10 +565,15 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); +// update flux + auto constituent_fluxes_so4_a2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a2)); + Kokkos::deep_copy(constituent_fluxes_so4_a2, + so4_a2SrfEmissData_out_.emiss_sectors[0]); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << std::endl; + << i <<":"< #include #include -#include +#include "share/grid/remap/abstract_remapper.hpp" +#include "share/io/scorpio_input.hpp" // For MAM4 aerosol configuration #include #include @@ -20,7 +21,8 @@ namespace scream { // The process responsible for handling MAM4 surface and online emissions. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; + using KT = ekat::KokkosTypes; + using view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -38,6 +40,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; + view_2d constituent_fluxes_; + public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -79,66 +83,71 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct Preprocess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data - void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere &wet_atm, - const mam_coupling::AerosolState &wet_aero, - const mam_coupling::DryAtmosphere &dry_atm, - const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; - wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; - dry_aero_pre_ = dry_aero; + void initialize(const view_2d &constituent_fluxes) { + constituent_fluxes_pre_ = constituent_fluxes; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { const int i = team.league_rank(); // column index - - compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, - dry_aero_pre_, i); + // zero-out the constituent surface fluxes for all gas and aerosol + // species. + for(auto icnst = mam4::utils::gasses_start_ind(); + icnst < mam4::aero_model::pcnst; ++icnst) { + constituent_fluxes_pre_(i, icnst) = 0; + } team.team_barrier(); } // operator() // local variables for preprocess struct - // number of horizontal columns and vertical levels - int ncol_pre_, nlev_pre_; + view_2d constituent_fluxes_pre_; - // local atmospheric and aerosol state data - mam_coupling::WetAtmosphere wet_atm_pre_; - mam_coupling::DryAtmosphere dry_atm_pre_; - mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMSrfOnlineEmiss::Preprocess private: // preprocessing scratch pad Preprocess preprocess_; - // IO structure to read in data for standard grids - + // Species index in tracer array with "pcnst" indices + enum class spcIndex_in_pcnst : int { + SO2 = 12, + DMS = 13, + so4_a1 = 15, + num_a1 = 22, + so4_a2 = 23, + num_a2 = 27, + pom_a4 = 36, + bc_a4 = 37, + num_a4 = 39, + }; + + //offset for converting pcnst index to gas_pcnst index + static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; + + // Data structures to read DMS data file std::shared_ptr dmsSrfEmissHorizInterp_; std::shared_ptr dmsSrfEmissDataReader_; srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - // Structures to store the data used for interpolation + // Data structures to read so2 data file std::shared_ptr so2SrfEmissHorizInterp_; std::shared_ptr so2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; + // Data structures to read bc_a4 data file std::shared_ptr bc_a4SrfEmissHorizInterp_; std::shared_ptr bc_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + // Data structures to read num_a1 data file std::shared_ptr num_a1SrfEmissHorizInterp_; std::shared_ptr num_a1SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; @@ -146,6 +155,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a1SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; + // Data structures to read num_a2 data file std::shared_ptr num_a2SrfEmissHorizInterp_; std::shared_ptr num_a2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; @@ -153,6 +163,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a2SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; + // Data structures to read num_a4 data file std::shared_ptr num_a4SrfEmissHorizInterp_; std::shared_ptr num_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; @@ -160,6 +171,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; + // Data structures to read pom_a4 data file std::shared_ptr pom_a4SrfEmissHorizInterp_; std::shared_ptr pom_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; @@ -167,6 +179,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { pom_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; + // Data structures to read so4_a1 data file std::shared_ptr so4_a1SrfEmissHorizInterp_; std::shared_ptr so4_a1SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; @@ -174,6 +187,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { so4_a1SrfEmissData_end_; srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; + // Data structures to read so4_a2 data file std::shared_ptr so4_a2SrfEmissHorizInterp_; std::shared_ptr so4_a2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp deleted file mode 100644 index 527efd5de18b..000000000000 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef MAM_EMISSIONS_READ_TABLES_HPP -#define MAM_EMISSIONS_READ_TABLES_HPP - -#include "ekat/ekat_parameter_list.hpp" -#include "mam_coupling.hpp" -#include "share/field/field_manager.hpp" -#include "share/grid/abstract_grid.hpp" -#include "share/grid/grids_manager.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_scorpio_interface.hpp" - -// later to mam_coupling.hpp -namespace scream::mam_coupling { - -using view_1d_host = typename KT::view_1d::HostMirror; -using view_1d_int_host = typename KT::view_1d::HostMirror; -using view_2d_host = typename KT::view_2d::HostMirror; -// using view_5d_host = typename KT::view_ND::HostMirror; -// using complex_view_1d = typename KT::view_1d>; - -constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; -constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; - -// FIXME: this will need to change when we remap to flattened column idx -constexpr int nlat_srf = 96; -constexpr int nlon_srf = 144; - -constexpr int nalti_online = 13; -constexpr int nlat_online = 96; -constexpr int nlon_online = 144; - -using namespace ShortFieldTagsNames; - -// std::map> map_srf_emiss_file_vars; - -inline void set_file_var_names(std::map> &var_map, - std::map &spec_map) { - // for (const auto &spec : spec_map) { - // std::string spec_name = spec.first; - // std::cout << "var_map[spec_name] = " << var_map[spec_name] << "\n"; - // } -} - -// struct AerosolSurfaceEmissionsHostData { -// // these have dim = n_species -// view_1d_host emis_species_index; -// view_1d_host emis_species_units; -// view_1d_host emis_species_name; -// // molecular weight -// view_1d_host emis_species_mw; -// // number of sectors in each field -// view_1d_int_host emis_species_nsectors; -// // FIXME: not quite sure what this does--maybe just a placeholder for -// // fields(:, i_sector)? -// view_1d_host emis_species_sector; -// // note fields have dim = n_species x nsectors -// // TODO: fields have units??? maybe the same as the upper spec units -// view_2d_host emis_species_fields; -// }; - -// using AerosolSurfaceEmissionsDeviceData = - // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; - -// inline void set_emissions_params( -// AerosolSurfaceEmissionsHostData& aerosol_emissions_host_data, -// ekat::ParameterList& params_emissions, -// std::map& layouts, -// std::map& host_views) { -// // Set up input structure to read data from file. -// using strvec_t = std::vector; -// // using namespace ShortFieldTagsNames; - -// // using SrfEmisDims = -// // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; SrfEmisDims -// // srf_emimssions_dims; -// } - -// inline void set_emissions_names(const std::map map_spec_id, -// const std::string emis_type, -// const ekat::ParameterList& m_params, -// std::map& host_views) { - -// using view_1d_host = typename KT::view_1d::HostMirror; - -// std::string - -// } // end set_emissions_names - -} // namespace scream::mam_coupling - -#endif diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b924706ed4ad..0934ad5fe7be 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -4,6 +4,20 @@ #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scorpio_input.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" +#include "share/scream_types.hpp" + +#include +#include +#include +#include +#include namespace scream::mam_coupling { namespace { From 89ec5e5ebfb178d2b83952259f491a489f1dceb6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 07:16:19 -0700 Subject: [PATCH 306/477] Removes num_sectors from the arg list --- ...and_online_emissions_process_interface.cpp | 92 ++++++++++--------- ...and_online_emissions_process_interface.hpp | 6 +- .../eamxx/src/physics/mam/srf_emission.hpp | 3 +- .../src/physics/mam/srf_emission_impl.hpp | 26 +++--- 4 files changed, 63 insertions(+), 64 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 21bf8a5bc1b8..5f396ef4df4e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -62,7 +62,7 @@ void MAMSrfOnlineEmiss::set_grids( std::array dms_sectors = {"DMS"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, dms_num_sectors, grid_, dms_data_file, dms_sectors, srf_map_file, + ncol_, grid_, dms_data_file, dms_sectors, srf_map_file, // output dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, dmsSrfEmissData_out_, dmsSrfEmissDataReader_); @@ -82,7 +82,7 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so2_num_sectors, grid_, so2_data_file, so2_sectors, srf_map_file, + ncol_, grid_, so2_data_file, so2_sectors, srf_map_file, // output so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_, so2SrfEmissDataReader_); @@ -102,8 +102,7 @@ void MAMSrfOnlineEmiss::set_grids( "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, bc_a4_num_sectors, grid_, bc_a4_data_file, bc_a4_sectors, - srf_map_file, + ncol_, grid_, bc_a4_data_file, bc_a4_sectors, srf_map_file, // output bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); @@ -123,8 +122,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a1_num_sectors, grid_, num_a1_data_file, num_a1_sectors, - srf_map_file, + ncol_, grid_, num_a1_data_file, num_a1_sectors, srf_map_file, // output num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, @@ -145,8 +143,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a2_num_sectors, grid_, num_a2_data_file, num_a2_sectors, - srf_map_file, + ncol_, grid_, num_a2_data_file, num_a2_sectors, srf_map_file, // output num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, @@ -170,8 +167,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a4_num_sectors, grid_, num_a4_data_file, num_a4_sectors, - srf_map_file, + ncol_, grid_, num_a4_data_file, num_a4_sectors, srf_map_file, // output num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, @@ -192,8 +188,7 @@ void MAMSrfOnlineEmiss::set_grids( "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, pom_a4_num_sectors, grid_, pom_a4_data_file, pom_a4_sectors, - srf_map_file, + ncol_, grid_, pom_a4_data_file, pom_a4_sectors, srf_map_file, // output pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, @@ -214,8 +209,7 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so4_a1_num_sectors, grid_, so4_a1_data_file, so4_a1_sectors, - srf_map_file, + ncol_, grid_, so4_a1_data_file, so4_a1_sectors, srf_map_file, // output so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, @@ -235,8 +229,7 @@ void MAMSrfOnlineEmiss::set_grids( std::array so4_a2_sectors = {"RCO", "TRA"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so4_a2_num_sectors, grid_, so4_a2_data_file, so4_a2_sectors, - srf_map_file, + ncol_, grid_, so4_a2_data_file, so4_a2_sectors, srf_map_file, // output so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, @@ -392,8 +385,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { dmsSrfEmissData_end_, dmsSrfEmissData_out_); // update flux - auto constituent_fluxes_DMS = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::DMS)); + auto constituent_fluxes_DMS = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); Kokkos::deep_copy(constituent_fluxes_DMS, dmsSrfEmissData_out_.emiss_sectors[0]); @@ -412,8 +406,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); // update flux - auto constituent_fluxes_SO2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::SO2)); + auto constituent_fluxes_SO2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::SO2)); Kokkos::deep_copy(constituent_fluxes_SO2, so2SrfEmissData_out_.emiss_sectors[0]); @@ -432,13 +427,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_bc_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::bc_a4)); + // update flux + auto constituent_fluxes_bc_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::bc_a4)); Kokkos::deep_copy(constituent_fluxes_bc_a4, bc_a4SrfEmissData_out_.emiss_sectors[0]); - //-------------------------------------------------------------------- // Interpolate num_a1 srf emiss data //-------------------------------------------------------------------- @@ -455,11 +450,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a1 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a1)); + // update flux + auto constituent_fluxes_num_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a1)); Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); + num_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a2 srf emiss data @@ -477,11 +473,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a2)); + // update flux + auto constituent_fluxes_num_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a2)); Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); + num_a2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a4 srf emiss data @@ -499,9 +496,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a4)); + // update flux + auto constituent_fluxes_num_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a4)); Kokkos::deep_copy(constituent_fluxes_num_a4, num_a4SrfEmissData_out_.emiss_sectors[0]); @@ -521,9 +519,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_pom_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::pom_a4)); + // update flux + auto constituent_fluxes_pom_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::pom_a4)); Kokkos::deep_copy(constituent_fluxes_pom_a4, pom_a4SrfEmissData_out_.emiss_sectors[0]); @@ -543,9 +542,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); -// update flux - auto constituent_fluxes_so4_a1 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a1)); + // update flux + auto constituent_fluxes_so4_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a1)); Kokkos::deep_copy(constituent_fluxes_so4_a1, so4_a1SrfEmissData_out_.emiss_sectors[0]); @@ -565,15 +565,17 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); -// update flux - auto constituent_fluxes_so4_a2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a2)); + // update flux + auto constituent_fluxes_so4_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a2)); Kokkos::deep_copy(constituent_fluxes_so4_a2, so4_a2SrfEmissData_out_.emiss_sectors[0]); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i <<":"< #include #include + #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" // For MAM4 aerosol configuration @@ -123,8 +124,9 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4 = 39, }; - //offset for converting pcnst index to gas_pcnst index - static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; + // offset for converting pcnst index to gas_pcnst index + static constexpr int offset_ = + mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; // Data structures to read DMS data file std::shared_ptr dmsSrfEmissHorizInterp_; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 0add3f343f9b..b1d45b2855eb 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -106,8 +106,7 @@ struct srfEmissFunctions { const ScalarT &t); template static void init_srf_emiss_objects( - const int ncol, const int num_sectors, - const std::shared_ptr &grid, + const int ncol, const std::shared_ptr &grid, const std::string &data_file, const std::array §ors, const std::string &srf_map_file, // output diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 0934ad5fe7be..4dc438b5a9aa 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -1,23 +1,20 @@ #ifndef SRF_EMISSION_IMPL_HPP #define SRF_EMISSION_IMPL_HPP +#include +#include +#include +#include +#include + +#include "physics/share/physics_constants.hpp" #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" #include "share/io/scorpio_input.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/grid/remap/coarsening_remapper.hpp" -#include "share/grid/remap/refining_remapper_p2p.hpp" -#include "share/grid/remap/identity_remapper.hpp" #include "share/io/scream_scorpio_interface.hpp" -#include "share/util/scream_timing.hpp" #include "share/scream_types.hpp" - -#include -#include -#include -#include -#include +#include "share/util/scream_timing.hpp" namespace scream::mam_coupling { namespace { @@ -285,8 +282,7 @@ void srfEmissFunctions::update_srfEmiss_timestate( template template void srfEmissFunctions::init_srf_emiss_objects( - const int ncol, const int num_sectors, - const std::shared_ptr &grid, + const int ncol, const std::shared_ptr &grid, const std::string &data_file, const std::array §ors, const std::string &srf_map_file, // output @@ -299,8 +295,8 @@ void srfEmissFunctions::init_srf_emiss_objects( create_horiz_remapper(grid, data_file, sectors, srf_map_file); // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, num_sectors); - SrfEmissData_end = srfEmissInput(ncol, num_sectors); + SrfEmissData_start = srfEmissInput(ncol, FN); + SrfEmissData_end = srfEmissInput(ncol, FN); SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) From 5db0b8a7033ce8ca178cd925cba92707b7f0366c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 16:34:13 -0700 Subject: [PATCH 307/477] Partial implementation of a vector for surface emissions --- ...and_online_emissions_process_interface.cpp | 35 ++++++ ...and_online_emissions_process_interface.hpp | 24 +++++ .../eamxx/src/physics/mam/srf_emission.hpp | 24 ++++- .../src/physics/mam/srf_emission_impl.hpp | 102 ++++++++++++++++-- 4 files changed, 174 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 5f396ef4df4e..d855a2929262 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -48,6 +48,41 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); + // EXPERIMENTAL + /*using srf_emission_variant = + std::variant, srf_emiss<2>, srf_emiss<4>, srf_emiss<6>, + srf_emiss<8>, srf_emiss<16>>; + + std::vector srfballivec; + + static constexpr int dms_num_sectors1 = 1; + srf_emiss dms1; + dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms1.sectors = {"DMS"}; + + srfballivec.push_back(dms1); + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, srfballivec[0].data_file , srfballivec[0].sectors, + srf_map_file, + // output + srfballivec[0].HorizInterp_, srfballivec[0].EmissData_start_, + srfballivec[0].EmissData_end_, srfballivec[0].EmissData_out_, + srfballivec[0].EmissDataReader_);*/ + + std::vector srfballivec; + srf_emiss dms1; + dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms1.sectors = {"DMS"}; + srfballivec.push_back(dms1); + + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, srfballivec[0].data_file, srfballivec[0].sectors, + srf_map_file, + // output + srfballivec[0].HorizInterp_, srfballivec[0].Data_start_, + srfballivec[0].Data_end_, srfballivec[0].Data_out_, + srfballivec[0].DataReader_); + //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 18c3712ed5f7..dca30a176ddb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -16,6 +16,7 @@ #include // #include #include +#include namespace scream { @@ -124,6 +125,29 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4 = 39, }; + // A struct carrying all the fields needed to read + // surface emissions of a species + // template + struct srf_emiss { + // srf_emiss(int n){numSectors=n;} + // data file name + std::string data_file; + // static int numSectors; + // Sector names in file + std::vector sectors; + + std::shared_ptr HorizInterp_; + std::shared_ptr DataReader_; + srfEmissFunc::srfEmissTimeState TimeState_; + srfEmissFunc::srfEmissInput Data_start_, Data_end_; + srfEmissFunc::srfEmissOutput Data_out_; + + /*srfEmissFunc::init_srf_emiss_objects( + // output + dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, + dmsSrfEmissData_out_, dmsSrfEmissDataReader_);*/ + }; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index b1d45b2855eb..585656b0ec1c 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,13 +66,18 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines - template + template static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &srfEmiss_data_file, - const std::array &field_names, + const std::array &field_names, const std::string &map_file); + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &srfEmiss_data_file, + const std::vector &field_names, const std::string &map_file); + static std::shared_ptr create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); @@ -104,10 +109,21 @@ struct srfEmissFunctions { KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, const ScalarX &x1, const ScalarT &t); - template + template + static void init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, + const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader); + static void init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, - const std::string &data_file, const std::array §ors, + const std::string &data_file, const std::vector §ors, const std::string &srf_map_file, // output std::shared_ptr &SrfEmissHorizInterp, diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 4dc438b5a9aa..b792c805b328 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -20,12 +20,12 @@ namespace scream::mam_coupling { namespace { template -template +template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &data_file, - const std::array §or_names, + const std::array §or_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -70,7 +70,70 @@ srfEmissFunctions::create_horiz_remapper( std::vector emiss_sectors; - for(int icomp = 0; icomp < FN; ++icomp) { + for(int icomp = 0; icomp < numSectors; ++icomp) { + auto comp_name = sector_names[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + f.allocate_view(); + emiss_sectors.push_back(f); + remapper->register_field_from_tgt(f); + } + + remapper->registration_ends(); + + return remapper; +} // create_horiz_remapper + +template +std::shared_ptr +srfEmissFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::vector §or_names, + const std::string &map_file) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit " + "the model. We only allow\n" + "srfEmiss data to be at the same or coarser resolution as " + "the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG( + map_file != "", + "ERROR: srfEmiss data is on a different grid than the model one,\n" + "but srfEmiss_remap_file is missing from srfEmiss parameter " + "list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + std::vector emiss_sectors; + + for(int icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); @@ -280,10 +343,35 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate template -template +template +void srfEmissFunctions::init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, + const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader) { + // Init horizontal remap + SrfEmissHorizInterp = + create_horiz_remapper(grid, data_file, sectors, srf_map_file); + + // Initialize the size of start/end/out data structures + SrfEmissData_start = srfEmissInput(ncol, numSectors); + SrfEmissData_end = srfEmissInput(ncol, numSectors); + SrfEmissData_out.init(ncol, 1, true); + + // Create reader (an AtmosphereInput object) + SrfEmissDataReader = + create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); +} // init_srf_emiss_objects + +template void srfEmissFunctions::init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, - const std::string &data_file, const std::array §ors, + const std::string &data_file, const std::vector §ors, const std::string &srf_map_file, // output std::shared_ptr &SrfEmissHorizInterp, @@ -295,8 +383,8 @@ void srfEmissFunctions::init_srf_emiss_objects( create_horiz_remapper(grid, data_file, sectors, srf_map_file); // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, FN); - SrfEmissData_end = srfEmissInput(ncol, FN); + SrfEmissData_start = srfEmissInput(ncol, sectors.size()); + SrfEmissData_end = srfEmissInput(ncol, sectors.size()); SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) From fecc309be5b91f42351ffd39c13f087910793f5e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 21:06:17 -0700 Subject: [PATCH 308/477] The vector for surface emissions is working in a for loop --- ...and_online_emissions_process_interface.cpp | 721 +++++++----------- ...and_online_emissions_process_interface.hpp | 17 +- 2 files changed, 279 insertions(+), 459 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index d855a2929262..eeb1b642cf24 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -48,227 +48,103 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); - // EXPERIMENTAL - /*using srf_emission_variant = - std::variant, srf_emiss<2>, srf_emiss<4>, srf_emiss<6>, - srf_emiss<8>, srf_emiss<16>>; - - std::vector srfballivec; - - static constexpr int dms_num_sectors1 = 1; - srf_emiss dms1; - dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms1.sectors = {"DMS"}; - - srfballivec.push_back(dms1); - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, srfballivec[0].data_file , srfballivec[0].sectors, - srf_map_file, - // output - srfballivec[0].HorizInterp_, srfballivec[0].EmissData_start_, - srfballivec[0].EmissData_end_, srfballivec[0].EmissData_out_, - srfballivec[0].EmissDataReader_);*/ - - std::vector srfballivec; - srf_emiss dms1; - dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms1.sectors = {"DMS"}; - srfballivec.push_back(dms1); - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, srfballivec[0].data_file, srfballivec[0].sectors, - srf_map_file, - // output - srfballivec[0].HorizInterp_, srfballivec[0].Data_start_, - srfballivec[0].Data_end_, srfballivec[0].Data_out_, - srfballivec[0].DataReader_); - //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string dms_data_file = - m_params.get("srf_emis_specifier_for_DMS"); - - // Number of sectors - static constexpr int dms_num_sectors = 1; - - // Sector names in file - std::array dms_sectors = {"DMS"}; + srf_emiss dms; - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, dms_data_file, dms_sectors, srf_map_file, - // output - dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, - dmsSrfEmissData_out_, dmsSrfEmissDataReader_); + // File name and sectors + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.sectors = {"DMS"}; + srf_emiss_species_.push_back(dms); // add to the vector //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so2_data_file = - m_params.get("srf_emis_specifier_for_SO2"); - - // Number of sectors - static constexpr int so2_num_sectors = 6; - - // Sector names in file - std::array so2_sectors = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so2_data_file, so2_sectors, srf_map_file, - // output - so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, - so2SrfEmissData_out_, so2SrfEmissDataReader_); - + srf_emiss so2; + // File name and sectors + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(so2); // add to the vector //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string bc_a4_data_file = - m_params.get("srf_emis_specifier_for_bc_a4"); - - // Number of sectors - static constexpr int bc_a4_num_sectors = 8; - - // Sector names in file - std::array bc_a4_sectors = { - "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, bc_a4_data_file, bc_a4_sectors, srf_map_file, - // output - bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + srf_emiss bc_a4; + // File name and sectors + bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(bc_a4); // add to the vector //-------------------------------------------------------------------- // Init num_a1 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a1_data_file = - m_params.get("srf_emis_specifier_for_num_a1"); - - // Number of sectors - static constexpr int num_a1_num_sectors = 4; - - // Sector names in file - std::array num_a1_sectors = { - "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a1_data_file, num_a1_sectors, srf_map_file, - // output - num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, - num_a1SrfEmissDataReader_); + srf_emiss num_a1; + // File name and sectors + num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; + srf_emiss_species_.push_back(num_a1); // add to the vector //-------------------------------------------------------------------- // Init num_a2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a2_data_file = - m_params.get("srf_emis_specifier_for_num_a2"); - - // Number of sectors - static constexpr int num_a2_num_sectors = 2; - - // Sector names in file - std::array num_a2_sectors = { - "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a2_data_file, num_a2_sectors, srf_map_file, - // output - num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, - num_a2SrfEmissDataReader_); + srf_emiss num_a2; + // File name and sectors + num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + srf_emiss_species_.push_back(num_a2); // add to the vector //-------------------------------------------------------------------- // Init num_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a4_data_file = - m_params.get("srf_emis_specifier_for_num_a4"); - - // Number of sectors - static constexpr int num_a4_num_sectors = 16; - - // Sector names in file - std::array num_a4_sectors = { - "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", - "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", - "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", - "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a4_data_file, num_a4_sectors, srf_map_file, - // output - num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, - num_a4SrfEmissDataReader_); + srf_emiss num_a4; + // File name and sectors + num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + srf_emiss_species_.push_back(num_a4); // add to the vector //-------------------------------------------------------------------- // Init pom_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string pom_a4_data_file = - m_params.get("srf_emis_specifier_for_pom_a4"); - - // Number of sectors - static constexpr int pom_a4_num_sectors = 8; - - // Sector names in file - std::array pom_a4_sectors = { - "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, pom_a4_data_file, pom_a4_sectors, srf_map_file, - // output - pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, - pom_a4SrfEmissDataReader_); + srf_emiss pom_a4; + // File name and sectors + pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); + pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(pom_a4); // add to the vector //-------------------------------------------------------------------- // Init so4_a1 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so4_a1_data_file = - m_params.get("srf_emis_specifier_for_so4_a1"); - - // Number of sectors - static constexpr int so4_a1_num_sectors = 4; - - // Sector names in file - std::array so4_a1_sectors = {"AGR", "SHP", - "SLV", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so4_a1_data_file, so4_a1_sectors, srf_map_file, - // output - so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, - so4_a1SrfEmissDataReader_); + srf_emiss so4_a1; + // File name and sectors + so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + srf_emiss_species_.push_back(so4_a1); //-------------------------------------------------------------------- // Init so4_a2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so4_a2_data_file = - m_params.get("srf_emis_specifier_for_so4_a2"); + srf_emiss so4_a2; + // File name and sectors + so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); + so4_a2.sectors = {"RCO", "TRA"}; + srf_emiss_species_.push_back(so4_a2); - // Number of sectors - static constexpr int so4_a2_num_sectors = 2; - - // Sector names in file - std::array so4_a2_sectors = {"RCO", "TRA"}; + //-------------------------------------------------------------------- + // Init data structures to read and interpolate + //-------------------------------------------------------------------- - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so4_a2_data_file, so4_a2_sectors, srf_map_file, - // output - so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, - so4_a2SrfEmissDataReader_); + for(const sr_emiss &ispec_srf : srf_emiss_species_) { + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, + // output + ispec_srf.HorizInterp_, ispec_srf.Data_start_, ispec_srf.Data_end_, + ispec_srf.Data_out_, ispec_srf.DataReader_); + } } // set_grid @@ -317,70 +193,12 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. //-------------------------------------------------------------------- - // Update dms srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - dmsSrfEmissDataReader_, timestamp(), curr_month, *dmsSrfEmissHorizInterp_, - dmsSrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, - so2SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update bc_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - bc_a4SrfEmissDataReader_, timestamp(), curr_month, - *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a1 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a1SrfEmissDataReader_, timestamp(), curr_month, - *num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a2SrfEmissDataReader_, timestamp(), curr_month, - *num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a4SrfEmissDataReader_, timestamp(), curr_month, - *num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update pom_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - pom_a4SrfEmissDataReader_, timestamp(), curr_month, - *pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so4_a1 srf emiss from file + // Update surface emissions from file //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so4_a1SrfEmissDataReader_, timestamp(), curr_month, - *so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so4_a2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so4_a2SrfEmissDataReader_, timestamp(), curr_month, - *so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_end_); - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) - << ":" << i << std::endl; + for(auto &ispec_srf : srf_emiss_species_) { + srfEmissFunc::update_srfEmiss_data_from_file( + ispec_srf.DataReader_, timestamp(), curr_month, *ispec_srf.HorizInterp_, + ispec_srf.Data_end_); } //----------------------------------------------------------------- @@ -407,211 +225,218 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- // Interpolate dms srf emiss data //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - dmsSrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - dmsSrfEmissDataReader_, ts, *dmsSrfEmissHorizInterp_, - dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_); + for(auto &ispec_srf : srf_emiss_species_) { + // Update srfEmissTimeState, note the addition of dt + ispec_srf.TimeState_.t_now = ts.frac_of_year_in_days(); - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, - dmsSrfEmissData_end_, dmsSrfEmissData_out_); + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + ispec_srf.DataReader_, ts, *ispec_srf.HorizInterp_, + ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_); - // update flux - auto constituent_fluxes_DMS = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(constituent_fluxes_DMS, - dmsSrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, - so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, - so2SrfEmissData_end_, so2SrfEmissData_out_); - // update flux - auto constituent_fluxes_SO2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::SO2)); - Kokkos::deep_copy(constituent_fluxes_SO2, - so2SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate bc_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, - bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_bc_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::bc_a4)); - Kokkos::deep_copy(constituent_fluxes_bc_a4, - bc_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a1 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, - num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, - num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a1)); - Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, - num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, - num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a2)); - Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, - num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, - num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a4)); - Kokkos::deep_copy(constituent_fluxes_num_a4, - num_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate pom_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, - pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, - pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_pom_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::pom_a4)); - Kokkos::deep_copy(constituent_fluxes_pom_a4, - pom_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so4_a1 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, - so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, - so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); - // update flux - auto constituent_fluxes_so4_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a1)); - Kokkos::deep_copy(constituent_fluxes_so4_a1, - so4_a1SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so4_a2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, - so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, - so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); - // update flux - auto constituent_fluxes_so4_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a2)); - Kokkos::deep_copy(constituent_fluxes_so4_a2, - so4_a2SrfEmissData_out_.emiss_sectors[0]); - - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] - << std::endl; + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, + ispec_srf.Data_end_, ispec_srf.Data_out_); + // update flux + auto clfx = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(cflx, dmsSrfEmissData_out_.emiss_sectors[0]); } + /* +// update flux +auto constituent_fluxes_DMS = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); +Kokkos::deep_copy(constituent_fluxes_DMS, + dmsSrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, + so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, + so2SrfEmissData_end_, so2SrfEmissData_out_); +// update flux +auto constituent_fluxes_SO2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::SO2)); +Kokkos::deep_copy(constituent_fluxes_SO2, + so2SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate bc_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, + bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_bc_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::bc_a4)); +Kokkos::deep_copy(constituent_fluxes_bc_a4, + bc_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a1 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, + num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, + num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a1)); +Kokkos::deep_copy(constituent_fluxes_num_a1, + num_a1SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, + num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, + num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a2)); +Kokkos::deep_copy(constituent_fluxes_num_a2, + num_a2SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, + num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, + num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a4)); +Kokkos::deep_copy(constituent_fluxes_num_a4, + num_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate pom_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, + pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, + pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_pom_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::pom_a4)); +Kokkos::deep_copy(constituent_fluxes_pom_a4, + pom_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so4_a1 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, + so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, + so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); +// update flux +auto constituent_fluxes_so4_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a1)); +Kokkos::deep_copy(constituent_fluxes_so4_a1, + so4_a1SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so4_a2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, + so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, + so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); +// update flux +auto constituent_fluxes_so4_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a2)); +Kokkos::deep_copy(constituent_fluxes_so4_a2, + so4_a2SrfEmissData_out_.emiss_sectors[0]); + +for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" + << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] + << std::endl; +}*/ /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index dca30a176ddb..2b321571da22 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -127,27 +127,22 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // A struct carrying all the fields needed to read // surface emissions of a species - // template struct srf_emiss { - // srf_emiss(int n){numSectors=n;} - // data file name + // Data file name std::string data_file; - // static int numSectors; - // Sector names in file + // Sector names in file std::vector sectors; - + // Data structure for reading interpolation std::shared_ptr HorizInterp_; std::shared_ptr DataReader_; srfEmissFunc::srfEmissTimeState TimeState_; srfEmissFunc::srfEmissInput Data_start_, Data_end_; srfEmissFunc::srfEmissOutput Data_out_; - - /*srfEmissFunc::init_srf_emiss_objects( - // output - dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, - dmsSrfEmissData_out_, dmsSrfEmissDataReader_);*/ }; + // A vector for carrying emissions for all the species + std::vector srf_emiss_species_; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; From 4f816563a6f71fa1edfe59b99be29825bf010ddb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 21:15:19 -0700 Subject: [PATCH 309/477] Fixes compilation issues in previous commit --- ...mam_srf_and_online_emissions_process_interface.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index eeb1b642cf24..17481dfc4d02 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -137,8 +137,7 @@ void MAMSrfOnlineEmiss::set_grids( //-------------------------------------------------------------------- // Init data structures to read and interpolate //-------------------------------------------------------------------- - - for(const sr_emiss &ispec_srf : srf_emiss_species_) { + for(srf_emiss &ispec_srf : srf_emiss_species_) { srfEmissFunc::init_srf_emiss_objects( ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, // output @@ -239,9 +238,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_, ispec_srf.Data_out_); // update flux - auto clfx = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(cflx, dmsSrfEmissData_out_.emiss_sectors[0]); + /*auto clfx = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(cflx, + dmsSrfEmissData_out_.emiss_sectors[0]);*/ } /* // update flux From f3aad49cac2c7a334b105b0fd73493b29b4c9f8d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 07:21:47 -0700 Subject: [PATCH 310/477] Updates constituent flux array with the righ units --- ...and_online_emissions_process_interface.cpp | 349 +++++------------- ...and_online_emissions_process_interface.hpp | 111 ++---- 2 files changed, 123 insertions(+), 337 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 17481dfc4d02..01167845fd5b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -51,98 +51,106 @@ void MAMSrfOnlineEmiss::set_grids( //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- - srf_emiss dms; - - // File name and sectors - dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms.sectors = {"DMS"}; + srf_emiss_ dms; + // File name, name and sectors + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.species_name = "dms"; + dms.sectors = {"DMS"}; srf_emiss_species_.push_back(dms); // add to the vector //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so2; - // File name and sectors - so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); - so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_ so2; + // File name, name and sectors + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.species_name = "so2"; + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(so2); // add to the vector //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss bc_a4; - // File name and sectors + srf_emiss_ bc_a4; + // File name, name and sectors bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); - bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + bc_a4.species_name = "bc_a4"; + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(bc_a4); // add to the vector //-------------------------------------------------------------------- // Init num_a1 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a1; - // File name and sectors + srf_emiss_ num_a1; + // File name, name and sectors num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); - num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", - "num_a1_SO4_WST"}; + num_a1.species_name = "num_a1"; + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; srf_emiss_species_.push_back(num_a1); // add to the vector //-------------------------------------------------------------------- // Init num_a2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a2; - // File name and sectors + srf_emiss_ num_a2; + // File name, name and sectors num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); - num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + num_a2.species_name = "num_a2"; + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; srf_emiss_species_.push_back(num_a2); // add to the vector //-------------------------------------------------------------------- // Init num_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a4; - // File name and sectors + srf_emiss_ num_a4; + // File name, name and sectors num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); - num_a4.sectors = { - "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", - "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", - "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", - "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + num_a4.species_name = "num_a4"; + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; srf_emiss_species_.push_back(num_a4); // add to the vector //-------------------------------------------------------------------- // Init pom_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss pom_a4; - // File name and sectors + srf_emiss_ pom_a4; + // File name, name and sectors pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); - pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + pom_a4.species_name = "pom_a4"; + pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(pom_a4); // add to the vector //-------------------------------------------------------------------- // Init so4_a1 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so4_a1; - // File name and sectors + srf_emiss_ so4_a1; + // File name, name and sectors so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); - so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + so4_a1.species_name = "so4_a1"; + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; srf_emiss_species_.push_back(so4_a1); //-------------------------------------------------------------------- // Init so4_a2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so4_a2; - // File name and sectors + srf_emiss_ so4_a2; + // File name, name and sectors so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); - so4_a2.sectors = {"RCO", "TRA"}; + so4_a2.species_name = "so4_a2"; + so4_a2.sectors = {"RCO", "TRA"}; srf_emiss_species_.push_back(so4_a2); //-------------------------------------------------------------------- // Init data structures to read and interpolate //-------------------------------------------------------------------- - for(srf_emiss &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::init_srf_emiss_objects( ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, // output - ispec_srf.HorizInterp_, ispec_srf.Data_start_, ispec_srf.Data_end_, - ispec_srf.Data_out_, ispec_srf.DataReader_); + ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, + ispec_srf.data_out_, ispec_srf.dataReader_); } } // set_grid @@ -183,6 +191,13 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); + // --------------------------------------------------------------- + // Allocate memory for local and work arrays + // --------------------------------------------------------------- + + // work array to store fluxes after unit conversions to kg/m2/s + fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); + // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; @@ -194,10 +209,10 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //-------------------------------------------------------------------- // Update surface emissions from file //-------------------------------------------------------------------- - for(auto &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::update_srfEmiss_data_from_file( - ispec_srf.DataReader_, timestamp(), curr_month, *ispec_srf.HorizInterp_, - ispec_srf.Data_end_); + ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, + ispec_srf.data_end_); } //----------------------------------------------------------------- @@ -218,232 +233,68 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + // policy to loop over columns only + const auto ncol_only_policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, 1); + // Gather time and state information for interpolation auto ts = timestamp() + dt; //-------------------------------------------------------------------- - // Interpolate dms srf emiss data + // Interpolate srf emiss data //-------------------------------------------------------------------- - for(auto &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { // Update srfEmissTimeState, note the addition of dt - ispec_srf.TimeState_.t_now = ts.frac_of_year_in_days(); + ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - ispec_srf.DataReader_, ts, *ispec_srf.HorizInterp_, - ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_); + ispec_srf.dataReader_, ts, *ispec_srf.horizInterp_, + ispec_srf.timeState_, ispec_srf.data_start_, ispec_srf.data_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, - ispec_srf.Data_end_, ispec_srf.Data_out_); - // update flux - /*auto clfx = + srfEmissFunc::srfEmiss_main(ispec_srf.timeState_, ispec_srf.data_start_, + ispec_srf.data_end_, ispec_srf.data_out_); + + //-------------------------------------------------------------------- + // Modify units to MKS units (from molecules/cm2/s to kg/m2/s) + //-------------------------------------------------------------------- + // Get species index in array with pcnst dimension (e.g., state_q or + // constituent_fluxes_) + const int species_index = spcIndex_in_pcnst_.at(ispec_srf.species_name); + + // modify units from molecules/cm2/s to kg/m2/s + const Real mfactor = + amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; + // Parallel loop over all the columns to update units + Kokkos::parallel_for( + ncol_only_policy, + KOKKOS_LAMBDA(const MAMSrfOnlineEmiss::KT::MemberType &team) { + const int icol = team.league_rank(); + fluxes_in_mks_units_(icol) = + ispec_srf.data_out_.emiss_sectors[0](icol) * mfactor; + }); + + // Get subview + auto flux_1d_view = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(cflx, - dmsSrfEmissData_out_.emiss_sectors[0]);*/ + spcIndex_in_pcnst_.at(ispec_srf.species_name)); + + // update flux in constituent_fluxes_ view + Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); + } + + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srf_emiss_species_[8].data_out_.emiss_sectors[0](i) + << ":" << i << ":" << constituent_fluxes_(i, 37) << std::endl; } - /* -// update flux -auto constituent_fluxes_DMS = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); -Kokkos::deep_copy(constituent_fluxes_DMS, - dmsSrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, - so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, - so2SrfEmissData_end_, so2SrfEmissData_out_); -// update flux -auto constituent_fluxes_SO2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::SO2)); -Kokkos::deep_copy(constituent_fluxes_SO2, - so2SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate bc_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, - bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_bc_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::bc_a4)); -Kokkos::deep_copy(constituent_fluxes_bc_a4, - bc_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a1 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, - num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, - num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a1)); -Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, - num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, - num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a2)); -Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, - num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, - num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a4)); -Kokkos::deep_copy(constituent_fluxes_num_a4, - num_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate pom_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, - pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, - pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_pom_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::pom_a4)); -Kokkos::deep_copy(constituent_fluxes_pom_a4, - pom_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so4_a1 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, - so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, - so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); -// update flux -auto constituent_fluxes_so4_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a1)); -Kokkos::deep_copy(constituent_fluxes_so4_a1, - so4_a1SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so4_a2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, - so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, - so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); -// update flux -auto constituent_fluxes_so4_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a2)); -Kokkos::deep_copy(constituent_fluxes_so4_a2, - so4_a2SrfEmissData_out_.emiss_sectors[0]); - -for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] - << std::endl; -}*/ /* Rough notes: Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions and - set_srf_emissions. + chemistry.F90. Basically call two subroutines, aero_model_emissions + and set_srf_emissions. Here is the code: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 2b321571da22..1669125b9fb5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -24,6 +24,7 @@ namespace scream { // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; using view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels @@ -44,6 +45,12 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { view_2d constituent_fluxes_; + // work array to store fluxes after unit conversions to kg/m2/s + view_1d fluxes_in_mks_units_; + + // Unified atomic mass unit used for unit conversion (BAD constant) + static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -112,110 +119,38 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; - // Species index in tracer array with "pcnst" indices - enum class spcIndex_in_pcnst : int { - SO2 = 12, - DMS = 13, - so4_a1 = 15, - num_a1 = 22, - so4_a2 = 23, - num_a2 = 27, - pom_a4 = 36, - bc_a4 = 37, - num_a4 = 39, - }; + // Species index (zero-based) in tracer array with "pcnst" dimension + // FIXME: Remove the hardwired indices and use a function + // to find them from an array. + const std::map spcIndex_in_pcnst_ = { + {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, + {"num_a1", 22}, {"so4_a2", 23}, {"num_a2", 27}, + {"pom_a4", 36}, {"bc_a4", 37}, {"num_a4", 39}}; // A struct carrying all the fields needed to read // surface emissions of a species - struct srf_emiss { + struct srf_emiss_ { + // species name + std::string species_name; // Data file name std::string data_file; // Sector names in file std::vector sectors; // Data structure for reading interpolation - std::shared_ptr HorizInterp_; - std::shared_ptr DataReader_; - srfEmissFunc::srfEmissTimeState TimeState_; - srfEmissFunc::srfEmissInput Data_start_, Data_end_; - srfEmissFunc::srfEmissOutput Data_out_; + std::shared_ptr horizInterp_; + std::shared_ptr dataReader_; + srfEmissFunc::srfEmissTimeState timeState_; + srfEmissFunc::srfEmissInput data_start_, data_end_; + srfEmissFunc::srfEmissOutput data_out_; }; // A vector for carrying emissions for all the species - std::vector srf_emiss_species_; + std::vector srf_emiss_species_; // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; - // Data structures to read DMS data file - std::shared_ptr dmsSrfEmissHorizInterp_; - std::shared_ptr dmsSrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; - srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; - srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - - // Data structures to read so2 data file - std::shared_ptr so2SrfEmissHorizInterp_; - std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; - - // Data structures to read bc_a4 data file - std::shared_ptr bc_a4SrfEmissHorizInterp_; - std::shared_ptr bc_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; - - // Data structures to read num_a1 data file - std::shared_ptr num_a1SrfEmissHorizInterp_; - std::shared_ptr num_a1SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; - - // Data structures to read num_a2 data file - std::shared_ptr num_a2SrfEmissHorizInterp_; - std::shared_ptr num_a2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; - - // Data structures to read num_a4 data file - std::shared_ptr num_a4SrfEmissHorizInterp_; - std::shared_ptr num_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; - - // Data structures to read pom_a4 data file - std::shared_ptr pom_a4SrfEmissHorizInterp_; - std::shared_ptr pom_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; - - // Data structures to read so4_a1 data file - std::shared_ptr so4_a1SrfEmissHorizInterp_; - std::shared_ptr so4_a1SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; - - // Data structures to read so4_a2 data file - std::shared_ptr so4_a2SrfEmissHorizInterp_; - std::shared_ptr so4_a2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so4_a2SrfEmissData_out_; - }; // MAMSrfOnlineEmiss } // namespace scream From 5b35e31791083141bd4e2ee18b4b54be2548d168 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 13:02:03 -0700 Subject: [PATCH 311/477] Adds constituent flixes interface and a test --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 231 ++++++++++++++++++ ...eamxx_mam_constituent_fluxes_interface.hpp | 76 ++++++ .../eamxx/src/physics/mam/mam_coupling.hpp | 18 ++ .../eamxx/src/physics/register_physics.hpp | 2 + .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../mam/constituent_fluxes/CMakeLists.txt | 44 ++++ .../mam/constituent_fluxes/input.yaml | 37 +++ .../mam/constituent_fluxes/output.yaml | 12 + 9 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 9138bba6d5a2..9874f79f9eb6 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -46,7 +46,8 @@ add_library(mam eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp eamxx_mam_wetscav_process_interface.cpp - eamxx_mam_srf_and_online_emissions_process_interface.cpp) + eamxx_mam_srf_and_online_emissions_process_interface.cpp + eamxx_mam_constituent_fluxes_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp new file mode 100644 index 000000000000..91d587c31d32 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -0,0 +1,231 @@ +#include +namespace scream { + +// ================================================================ +// Constructor +// ================================================================ +MAMConstituentFluxes::MAMConstituentFluxes(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ================================================================ +// SET_GRIDS +// ================================================================ +void MAMConstituentFluxes::set_grids( + const std::shared_ptr grids_manager) { + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + + static constexpr int pcnst = mam4::aero_model::pcnst; + + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + static constexpr Units m2(m * m, "m2"); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + grid_name); + + // --------------------------------------------------------------------- + // These variables are "Updated" or inputs/outputs for the process + // --------------------------------------------------------------------- + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + +} // set_grid + +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMConstituentFluxes::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializes the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMConstituentFluxes::init_buffers( + const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMConstituentFluxes."); +} + +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ +void MAMConstituentFluxes::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Inputs + // --------------------------------------------------------------- + constituent_fluxes_ = + get_field_in("constituent_fluxes").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + //----------------------------------------------------------------- + // Allocate memory + //----------------------------------------------------------------- + + rpdel_ = view_2d("rpdel_", ncol_, nlev_); + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + // preprocess_.initialize(constituent_fluxes_); + +} // end initialize_impl() + +// ================================================================ +// RUN_IMPL +// ================================================================ +void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + /*FORTRAN CODE: + What we need: + 1. ncol + 2. all tracers + 3. timestep (verify what is rztodt??) + 4. gravit, rpdel + 5. cflx + 6. Ensure dry/et-wet/dry isconsistent....comment on that!! + + + ncol = state%ncol + + !------------------------------------------------------- + ! Assume 'wet' mixing ratios in surface diffusion code. + ! don't convert co2 tracers to wet mixing ratios + + cnst_type_loc(:) = cnst_type(:) + call set_dry_to_wet(state, cnst_type_loc) + + !------------------------------------------------------- + ! Initialize ptend + + lq(:) = .TRUE. + call physics_ptend_init(ptend, state%psetcols, 'clubb_srf', lq=lq) + + !------------------------------------------------------- + ! Calculate tracer mixing ratio tendencies from cflx + + rztodt = 1._r8/ztodt + ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) + tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver) + + do m = 2, pcnst + ptend%q(:ncol,pver,m) = ptend%q(:ncol,pver,m) + tmp1(:ncol) * + cam_in%cflx(:ncol,m) enddo + + ptend%q(:ncol,:pver,:) = (ptend%q(:ncol,:pver,:) - state%q(:ncol,:pver,:)) * + rztodt + + ! Convert tendencies of dry constituents to dry basis. + do m = 1,pcnst + if (cnst_type(m).eq.'dry') then + ptend%q(:ncol,:pver,m) = + ptend%q(:ncol,:pver,m)*state%pdel(:ncol,:pver)/state%pdeldry(:ncol,:pver) + endif + end do + + !------------------------------------------------------- + ! convert wet mmr back to dry before conservation check + ! avoid converting co2 tracers again + + cnst_type_loc(:) = cnst_type(:) + call co2_cycle_set_cnst_type(cnst_type_loc, 'wet') + call set_wet_to_dry(state, cnst_type_loc) + + */ +} + +// ============================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp new file mode 100644 index 000000000000..0f17e5b0632a --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -0,0 +1,76 @@ +#ifndef EAMXX_MAM_CONSTITUENT_FLUXES_HPP +#define EAMXX_MAM_CONSTITUENT_FLUXES_HPP + +// For declaring contituent fluxes class derived from atm process +// class +#include +// For MAM4 aerosol configuration +#include +#include + +namespace scream { + +// The process responsible for applying MAM4 constituent fluxes. The +// AD stores exactly ONE instance of this class in its list of subcomponents. +class MAMConstituentFluxes final : public scream::AtmosphereProcess { + using KT = ekat::KokkosTypes; + using view_2d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + + // number of horizontal columns + int ncol_, nlev_; + + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + const_view_2d constituent_fluxes_; + + view_2d rpdel_; // Inverse of pdel_ or pseudo_density + + public: + // Constructor + MAMConstituentFluxes(const ekat::Comm &comm, + const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_constituent_fluxes"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMConstituentFluxes + +} // namespace scream + +#endif // EAMXX_MAM_CONSTITUENT_FLUXES_HPP diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index d490ab761559..09d64d6028d9 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -776,6 +776,24 @@ void compute_wet_mixing_ratios(const Team& team, }); } +// Computes the reciprocal of pseudo density for a column +KOKKOS_INLINE_FUNCTION +void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, + const_view_2d pdel, + const int nlev, + // output + view_2d rpdel) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); + }); +} // Scream (or EAMxx) can sometimes extend views beyond model levels (nlev) as it uses // "packs". Following function copies a 2d view till model levels inline diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index f8d3a9edb199..d837e96311f4 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -30,6 +30,7 @@ #include "physics/mam/eamxx_mam_aci_process_interface.hpp" #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" #include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" +#include "physics/mam/eamxx_mam_constituent_fluxes_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -70,6 +71,7 @@ inline void register_physics () { proc_factory.register_product("mam4_aci",&create_atmosphere_process); proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); proc_factory.register_product("mam4_srf_online_emiss",&create_atmosphere_process); + proc_factory.register_product("mam4_constituent_fluxes",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index 62435db229fe..40565d94bf8e 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -24,6 +24,7 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam/drydep) add_subdirectory(mam/wet_scav) add_subdirectory(mam/emissions) + add_subdirectory(mam/constituent_fluxes) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt new file mode 100644 index 000000000000..2615f152b264 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt @@ -0,0 +1,44 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_constituent_fluxes_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_constituent_fluxes physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_constituent_fluxes physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml new file mode 100644 index 000000000000..d012274d1469 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml @@ -0,0 +1,37 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_constituent_fluxes] +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + pbl_height : 1.0 + constituent_fluxes: 1.e-5 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml new file mode 100644 index 000000000000..e51b96eb5ce3 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml @@ -0,0 +1,12 @@ +%YAML 1.1 +--- +filename_prefix: mam4_constituent_fluxes_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - T_mid +output_control: + Frequency: 1 + frequency_units: nsteps +... From b8ff3601be8c9ceb91a08a63536405645b7226b9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 19:39:46 -0700 Subject: [PATCH 312/477] Adds code to create progs and atm datastructures --- ...eamxx_mam_constituent_fluxes_interface.cpp | 190 ++++++++++++++++-- 1 file changed, 176 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 91d587c31d32..6075e066b54d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -26,8 +26,11 @@ void MAMConstituentFluxes::set_grids( using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers auto n_unit = 1 / kg; // units of number mixing ratios of tracers + auto nondim = ekat::units::Units::nondimensional(); + FieldLayout scalar2d = grid_->get_2d_scalar_layout(); FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); static constexpr int pcnst = mam4::aero_model::pcnst; @@ -37,44 +40,98 @@ void MAMConstituentFluxes::set_grids( // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process // -------------------------------------------------------------------------- + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg](Require only for building DS) + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg](Require only for building DS) + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg](Require only for building DS) + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) + add_field("omega", scalar3d_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + // Layer thickness(pdel) [Pa] at midpoints add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + // Planetary boundary layer height [m] (Require only for building DS) + add_field("pbl_height", scalar2d, m, grid_name); + + // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + static constexpr Units m2(m * m, "m2"); + static constexpr Units s2(s * s, "s2"); + + // Surface geopotential [m2/s2] (Require only for building DS) + add_field("phis", scalar2d, m2 / s2, grid_name); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); // --------------------------------------------------------------------- // These variables are "Updated" or inputs/outputs for the process // --------------------------------------------------------------------- - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - + // NOTE: Cloud borne aerosols are not updated in this process but are included + // to create datastructures. + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - + mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } - } - } + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(mode, a); + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); + } + } // end for loop num species + } // end for loop for num modes - // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); - } + } // end for loop num gases } // set_grid @@ -111,8 +168,42 @@ void MAMConstituentFluxes::init_buffers( // ================================================================ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- - // Inputs + // Input fields read in from IC file, namelist or other processes // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + // FIMXE: specifically look which among these are actually used by the process + wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + constituent_fluxes_ = get_field_in("constituent_fluxes").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); @@ -126,6 +217,12 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = @@ -136,6 +233,15 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } } } for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { @@ -167,8 +273,65 @@ void MAMConstituentFluxes::run_impl(const double dt) { // preprocess input -- needs a scan for the calculation of atm height // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); // Kokkos::fence(); + + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] + + mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, + nlev_, + // output + rpdel_); + // ------------------------------------------------------------------- + // NOTES: The following code is an adaptation of cflx.F90 code in E3SM + // In EAMxx, all constituents are considered "wet" (or have wet mixing + // ratios), so we are *not* doing any wet to dry conversions here. We + // are simply updating the MAM4xx tracers using the "constituent + // fluxes" + // ------------------------------------------------------------------- + + Real rztodt = 1.0 / dt; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + /////////dddd + // To construct state_q, we need fields from Prognostics (aerosols) + // and Atmosphere (water species such as qv, qc etc.) + + // get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(dry_aero_, icol); + + // get atmospheric quantities + haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); + + // Construct state_q (interstitial) and qqcw (cloud borne) arrays + /*constexpr auto pver = mam4::ndrop::pver; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pver), [&](int klev) + { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics(progs_at_col, haero_atm, + state_q_at_lev_col, + klev); + + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } + });*/ + }); + ////////dddd + + /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) + tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ + /*FORTRAN CODE: What we need: 1. ncol @@ -183,7 +346,6 @@ void MAMConstituentFluxes::run_impl(const double dt) { !------------------------------------------------------- ! Assume 'wet' mixing ratios in surface diffusion code. - ! don't convert co2 tracers to wet mixing ratios cnst_type_loc(:) = cnst_type(:) call set_dry_to_wet(state, cnst_type_loc) From 552a785b600e3222d7f0e0ba263190f2e9761066 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 21:31:38 -0700 Subject: [PATCH 313/477] A working end-to-end code --- .../physics/mam/eamxx_mam_aci_functions.hpp | 2 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 58 +++++++++---------- ...eamxx_mam_constituent_fluxes_interface.hpp | 2 +- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 03b04ac739a0..ab142fedc78f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -397,7 +397,7 @@ void call_function_dropmixnuc( progs_at_col, haero_atm, state_q_at_lev_col, klev); // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + int istart = mam4::utils::aero_start_ind(); // create colum views of state_q for(int icnst = istart; icnst < mam4::aero_model::pcnst; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 6075e066b54d..54d863f8b6fb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -215,13 +215,11 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); - dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); - dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios @@ -231,7 +229,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -240,7 +237,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { if(strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); - dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } @@ -248,7 +244,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); - dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } //----------------------------------------------------------------- @@ -289,9 +284,6 @@ void MAMConstituentFluxes::run_impl(const double dt) { // are simply updating the MAM4xx tracers using the "constituent // fluxes" // ------------------------------------------------------------------- - - Real rztodt = 1.0 / dt; - Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -301,33 +293,41 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get prognostics mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(dry_aero_, icol); + mam_coupling::aerosols_for_column(wet_aero_, icol); // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays - /*constexpr auto pver = mam4::ndrop::pver; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pver), [&](int klev) - { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics(progs_at_col, haero_atm, - state_q_at_lev_col, - klev); - - // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // create colum views of state_q - for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - } - });*/ + constexpr auto pver = mam4::ndrop::pver; + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, pver), [&](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); + std::cout << "istart:" << mam4::utils::gasses_start_ind() + << std::endl; + // kg/m2/s to kg/kg/ + Real unit_factor = dt * gravit * rpdel_(icol, klev); + // Update state with the constituent fluxes (modified + // units) + for(int icnst = istart; icnst < mam4::aero_model::pcnst; + ++icnst) { + state_q_at_lev_col[icnst] = + state_q_at_lev_col[icnst] + + constituent_fluxes_(icol, icnst) * unit_factor; + } + mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, + progs_at_col, klev); + }); }); - ////////dddd /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 0f17e5b0632a..02c4b8790938 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -25,7 +25,7 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { mam_coupling::DryAtmosphere dry_atm_; // aerosol state variables - mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::AerosolState wet_aero_; //, dry_aero_; // buffer for sotring temporary variables mam_coupling::Buffer buffer_; From a3c0ab9dbf0c08c31407db8a31f316e92107160f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 22:20:52 -0700 Subject: [PATCH 314/477] Verified change in DMS conc before and after update, moved aci rpdel function to mam_coupling --- .../physics/mam/eamxx_mam_aci_functions.hpp | 17 --- .../mam/eamxx_mam_aci_process_interface.cpp | 2 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 117 ++++++------------ ...eamxx_mam_constituent_fluxes_interface.hpp | 41 ++++++ 4 files changed, 78 insertions(+), 99 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index ab142fedc78f..8bb52c918d23 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -198,23 +198,6 @@ void store_liquid_cloud_fraction( }); } -void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, - MAMAci::const_view_2d pdel, - const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { - EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), - "Error: pdel should be > 0.\n"); - rpdel(icol, kk) = 1 / pdel(icol, kk); - }); - }); -} - void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, const Real dt, mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e919aa066b47..7800a43c252d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -597,7 +597,7 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); - compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, + mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 54d863f8b6fb..5c5129d1b7c1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -84,6 +84,7 @@ void MAMConstituentFluxes::set_grids( // Surface geopotential [m2/s2] (Require only for building DS) add_field("phis", scalar2d, m2 / s2, grid_name); + // Constituent fluxes at the surface (gasses and aerosols) add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); @@ -91,7 +92,8 @@ void MAMConstituentFluxes::set_grids( // These variables are "Updated" or inputs/outputs for the process // --------------------------------------------------------------------- // NOTE: Cloud borne aerosols are not updated in this process but are included - // to create datastructures. + // to create data structures. + // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { @@ -191,6 +193,7 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; @@ -204,9 +207,9 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { dry_atm_.w_updraft = buffer_.w_updraft; dry_atm_.z_surf = 0.0; // FIXME: for now + // Constituent fluxes at the surface (gasses and aerosols) [kg/m2/s] constituent_fluxes_ = get_field_in("constituent_fluxes").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -249,12 +252,13 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- // Allocate memory //----------------------------------------------------------------- - + // Inverse of pseudo_density (1/Pa or or N/m2 or kg/m/s2) rpdel_ = view_2d("rpdel_", ncol_, nlev_); + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- - // preprocess_.initialize(constituent_fluxes_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); } // end initialize_impl() @@ -265,29 +269,39 @@ void MAMConstituentFluxes::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + // preprocess input -- needs a scan for the calculation of atm height - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + // policy haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] + // compute inverse of pseudo_density mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); - // ------------------------------------------------------------------- - // NOTES: The following code is an adaptation of cflx.F90 code in E3SM - // In EAMxx, all constituents are considered "wet" (or have wet mixing - // ratios), so we are *not* doing any wet to dry conversions here. We - // are simply updating the MAM4xx tracers using the "constituent - // fluxes" - // ------------------------------------------------------------------- + std::cout << "BEFORE:::" << wet_aero_.gas_mmr[4](1, 1) << std::endl; + // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - /////////dddd + // To construct state_q, we need fields from Prognostics (aerosols) // and Atmosphere (water species such as qv, qc etc.) @@ -298,10 +312,9 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); - // Construct state_q (interstitial) and qqcw (cloud borne) arrays - constexpr auto pver = mam4::ndrop::pver; + // Construct state_q (interstitial) array Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, pver), [&](int klev) { + Kokkos::TeamVectorRange(team, nlev_), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -312,16 +325,15 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - std::cout << "istart:" << mam4::utils::gasses_start_ind() - << std::endl; - // kg/m2/s to kg/kg/ + + // Factor to convert units from kg/m2/s to kg/kg Real unit_factor = dt * gravit * rpdel_(icol, klev); + // Update state with the constituent fluxes (modified // units) for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_at_lev_col[icnst] = - state_q_at_lev_col[icnst] + + state_q_at_lev_col[icnst] += constituent_fluxes_(icol, icnst) * unit_factor; } mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, @@ -329,65 +341,8 @@ void MAMConstituentFluxes::run_impl(const double dt) { }); }); - /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) - tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ - - /*FORTRAN CODE: - What we need: - 1. ncol - 2. all tracers - 3. timestep (verify what is rztodt??) - 4. gravit, rpdel - 5. cflx - 6. Ensure dry/et-wet/dry isconsistent....comment on that!! - - - ncol = state%ncol - - !------------------------------------------------------- - ! Assume 'wet' mixing ratios in surface diffusion code. - - cnst_type_loc(:) = cnst_type(:) - call set_dry_to_wet(state, cnst_type_loc) - - !------------------------------------------------------- - ! Initialize ptend - - lq(:) = .TRUE. - call physics_ptend_init(ptend, state%psetcols, 'clubb_srf', lq=lq) - - !------------------------------------------------------- - ! Calculate tracer mixing ratio tendencies from cflx - - rztodt = 1._r8/ztodt - ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) - tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver) - - do m = 2, pcnst - ptend%q(:ncol,pver,m) = ptend%q(:ncol,pver,m) + tmp1(:ncol) * - cam_in%cflx(:ncol,m) enddo - - ptend%q(:ncol,:pver,:) = (ptend%q(:ncol,:pver,:) - state%q(:ncol,:pver,:)) * - rztodt - - ! Convert tendencies of dry constituents to dry basis. - do m = 1,pcnst - if (cnst_type(m).eq.'dry') then - ptend%q(:ncol,:pver,m) = - ptend%q(:ncol,:pver,m)*state%pdel(:ncol,:pver)/state%pdeldry(:ncol,:pver) - endif - end do - - !------------------------------------------------------- - ! convert wet mmr back to dry before conservation check - ! avoid converting co2 tracers again - - cnst_type_loc(:) = cnst_type(:) - call co2_cycle_set_cnst_type(cnst_type_loc, 'wet') - call set_wet_to_dry(state, cnst_type_loc) - - */ -} + std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; +} // run_impl ends // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 02c4b8790938..81c6ee529857 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -69,6 +69,47 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::DryAtmosphere &dry_atm) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + dry_atm_pre_ = dry_atm; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + }; // Preprocess + + private: + // preprocessing scratch pads + Preprocess preprocess_; + }; // MAMConstituentFluxes } // namespace scream From 3496c7ddccec197485379e0d96cd70f7e34d110b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 06:53:57 -0700 Subject: [PATCH 315/477] Cleanup of some unused hpp file includes and some variables --- ...eamxx_mam_constituent_fluxes_interface.hpp | 1 + ...and_online_emissions_process_interface.cpp | 17 ++++++------- ...and_online_emissions_process_interface.hpp | 25 ++++++------------- .../single-process/mam/emissions/output.yaml | 2 +- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 81c6ee529857..448ae5464b87 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -4,6 +4,7 @@ // For declaring contituent fluxes class derived from atm process // class #include + // For MAM4 aerosol configuration #include #include diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 01167845fd5b..1fc3ddb4b685 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,4 +1,4 @@ -#include +//#include #include namespace scream { @@ -10,7 +10,7 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be - * initialized here. Like universal constants, mam wetscav options. + * initialized here. Like universal constants. */ } @@ -22,7 +22,8 @@ void MAMSrfOnlineEmiss::set_grids( grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); @@ -32,16 +33,11 @@ void MAMSrfOnlineEmiss::set_grids( const FieldLayout scalar2d_pcnct = grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); - // -------------------------------------------------------------------------- - // These variables are "Required" or pure inputs for the process - // -------------------------------------------------------------------------- - // Temperature[K] at midpoints - add_field("T_mid", scalar3d_mid, K, grid_name); - // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- static constexpr Units m2(m * m, "m2"); + // Constituent fluxes of species in [kg/m2/s] add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); @@ -189,6 +185,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- // Output fields // --------------------------------------------------------------- + // Constituent fluxes of species in [kg/m2/s] constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); // --------------------------------------------------------------- @@ -227,7 +224,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, 1); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1669125b9fb5..b67b1de0bd7d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,22 +1,19 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP -#include -#include -#include - #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" + // For MAM4 aerosol configuration #include #include -#include + // For declaring surface and online emission class derived from atm process // class #include + // #include #include -#include namespace scream { @@ -30,22 +27,16 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; - // Wet and dry states of atmosphere - mam_coupling::WetAtmosphere wet_atm_; - mam_coupling::DryAtmosphere dry_atm_; - - // aerosol state variables - mam_coupling::AerosolState wet_aero_, dry_aero_; - // buffer for sotring temporary variables mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; + // Constituent fluxes of species in [kg/m2/s] view_2d constituent_fluxes_; - // work array to store fluxes after unit conversions to kg/m2/s + // Work array to store fluxes after unit conversions to kg/m2/s view_1d fluxes_in_mks_units_; // Unified atomic mass unit used for unit conversion (BAD constant) @@ -54,9 +45,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { public: using srfEmissFunc = mam_coupling::srfEmissFunctions; - template - using uview_2d = Unmanaged>; - // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -132,10 +120,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct srf_emiss_ { // species name std::string species_name; + // Data file name std::string data_file; + // Sector names in file std::vector sectors; + // Data structure for reading interpolation std::shared_ptr horizInterp_; std::shared_ptr dataReader_; diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml index b6e2d545d3ca..433f7d58fa40 100644 --- a/components/eamxx/tests/single-process/mam/emissions/output.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -5,7 +5,7 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid + - constituent_fluxes output_control: Frequency: 1 frequency_units: nsteps From 4048ae83f28f9e32b19d195b8c055023a3496ebe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 07:49:58 -0700 Subject: [PATCH 316/477] Removes unused variables --- ...and_online_emissions_process_interface.cpp | 64 +++---------------- .../src/physics/mam/srf_emission_impl.hpp | 16 +---- 2 files changed, 13 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 1fc3ddb4b685..91a5b161dff4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -26,10 +26,8 @@ void MAMSrfOnlineEmiss::set_grids( nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; - FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); static constexpr int pcnst = mam4::aero_model::pcnst; - const FieldLayout scalar2d_pcnct = grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); @@ -149,7 +147,7 @@ void MAMSrfOnlineEmiss::set_grids( ispec_srf.data_out_, ispec_srf.dataReader_); } -} // set_grid +} // set_grid ends // ================================================================ // REQUEST_BUFFER_SIZE_IN_BYTES @@ -192,16 +190,16 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Allocate memory for local and work arrays // --------------------------------------------------------------- - // work array to store fluxes after unit conversions to kg/m2/s + // Work array to store fluxes after unit conversions to kg/m2/s fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; - // Load the first month into srfEmiss_end. + // Load the first month into data_end. - // Note: At the first time step, the data will be moved into srfEmiss_beg, - // and srfEmiss_end will be reloaded from file with the new month. + // Note: At the first time step, the data will be moved into data_beg, + // and data_end will be reloaded from file with the new month. //-------------------------------------------------------------------- // Update surface emissions from file @@ -209,7 +207,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::update_srfEmiss_data_from_file( ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, - ispec_srf.data_end_); + ispec_srf.data_end_); // output } //----------------------------------------------------------------- @@ -242,12 +240,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- for(srf_emiss_ &ispec_srf : srf_emiss_species_) { - // Update srfEmissTimeState, note the addition of dt + // Update TimeState, note the addition of dt ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( ispec_srf.dataReader_, ts, *ispec_srf.horizInterp_, + // output ispec_srf.timeState_, ispec_srf.data_start_, ispec_srf.data_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. @@ -280,51 +279,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // update flux in constituent_fluxes_ view Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); - } - - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srf_emiss_species_[8].data_out_.emiss_sectors[0](i) - << ":" << i << ":" << constituent_fluxes_(i, 37) << std::endl; - } - - /* Rough notes: - - Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions - and set_srf_emissions. - - Here is the code: - - ! initialize chemistry constituent surface fluxes to zero - do m = 2,pcnst - n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 - enddo - - ! aerosol emissions ... - call aero_model_emissions( state, & ! in - cam_in ) ! out - - ! prescribed emissions from file ... - - !----------------------------------------------------------------------- - ! ... Set surface emissions - !----------------------------------------------------------------------- - call set_srf_emissions( lchnk, ncol, sflx(:,:) ) - - do m = 1,pcnst - n = map2chm(m) - if ( n /= h2o_ndx .and. n > 0 ) then - cam_in%cflx(:ncol,m) = cam_in%cflx(:ncol,m) + sflx(:ncol,n) - call outfld( sflxnam(m), cam_in%cflx(:ncol,m), ncol,lchnk ) - endif - enddo - - - */ - - std::cout << "End of surface emissions run" << std::endl; -} + } // for loop for species +} // run_imple ends // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b792c805b328..3fa458450a3d 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -1,20 +1,9 @@ #ifndef SRF_EMISSION_IMPL_HPP #define SRF_EMISSION_IMPL_HPP -#include -#include -#include -#include -#include - -#include "physics/share/physics_constants.hpp" -#include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" -#include "share/io/scorpio_input.hpp" #include "share/io/scream_scorpio_interface.hpp" -#include "share/scream_types.hpp" -#include "share/util/scream_timing.hpp" namespace scream::mam_coupling { namespace { @@ -297,8 +286,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Read fields from the file for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { - auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], aa); + auto sector = + srfEmiss_horiz_interp.get_tgt_field(i).get_view(); + Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], sector); } Kokkos::fence(); From 0b9b81d2875b838cec329926e61910a3dcd379fe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 11:55:58 -0700 Subject: [PATCH 317/477] Cleanup - removed unused codes and added kokkos reduction loop --- .../eamxx/src/physics/mam/srf_emission.hpp | 5 + .../src/physics/mam/srf_emission_impl.hpp | 143 +++--------------- 2 files changed, 25 insertions(+), 123 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 585656b0ec1c..82a4b3113821 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -8,6 +8,11 @@ namespace { template struct srfEmissFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + struct srfEmissTimeState { srfEmissTimeState() = default; // Whether the timestate has been initialized. diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 3fa458450a3d..ef99879277eb 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -8,71 +8,6 @@ namespace scream::mam_coupling { namespace { -template -template -std::shared_ptr -srfEmissFunctions::create_horiz_remapper( - const std::shared_ptr &model_grid, - const std::string &data_file, - const std::array §or_names, - const std::string &map_file) { - using namespace ShortFieldTagsNames; - - scorpio::register_file(data_file, scorpio::Read); - const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); - scorpio::release_file(data_file); - - // We could use model_grid directly if using same num levels, - // but since shallow clones are cheap, we may as well do it (less lines of - // code) - auto horiz_interp_tgt_grid = - model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); - - const int ncols_model = model_grid->get_num_global_dofs(); - std::shared_ptr remapper; - if(ncols_data == ncols_model) { - remapper = std::make_shared( - horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); - } else { - EKAT_REQUIRE_MSG(ncols_data <= ncols_model, - "Error! We do not allow to coarsen srfEmiss data to fit " - "the model. We only allow\n" - "srfEmiss data to be at the same or coarser resolution as " - "the model.\n"); - // We must have a valid map file - EKAT_REQUIRE_MSG( - map_file != "", - "ERROR: srfEmiss data is on a different grid than the model one,\n" - "but srfEmiss_remap_file is missing from srfEmiss parameter " - "list."); - - remapper = - std::make_shared(horiz_interp_tgt_grid, map_file); - } - - remapper->registration_begins(); - - const auto tgt_grid = remapper->get_tgt_grid(); - - const auto layout_2d = tgt_grid->get_2d_scalar_layout(); - const auto nondim = ekat::units::Units::nondimensional(); - - std::vector emiss_sectors; - - for(int icomp = 0; icomp < numSectors; ++icomp) { - auto comp_name = sector_names[icomp]; - // set and allocate fields - Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); - f.allocate_view(); - emiss_sectors.push_back(f); - remapper->register_field_from_tgt(f); - } - - remapper->registration_ends(); - - return remapper; -} // create_horiz_remapper - template std::shared_ptr srfEmissFunctions::create_horiz_remapper( @@ -186,41 +121,29 @@ void srfEmissFunctions::perform_time_interpolation( std::to_string(t_beg) + "\n delta_t: " + std::to_string(delta_t) + "\n"); - for(int icol = 0; icol < data_beg.data.ncols; ++icol) { - Real accum = 0; - for(int i = 0; i < data_beg.data.nsectors; ++i) { - accum += - linear_interp(data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), delta_t_fraction); - if(icol == 19) std::cout << "accum:" << accum << std::endl; - } - data_out.emiss_sectors[0](icol) = accum; - } - - /*const auto policy = ESU::get_default_team_policy(data_beg.data.ncols, 1); + const int nsectors = data_beg.data.nsectors; + const int ncols = data_beg.data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nsectors); Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - double result; - - Kokkos::parallel_reduce("Loop1", N, KOKKOS_LAMBDA (const int& i, double& - lsum) { lsum += 1.0*i; - }, result); - - Kokkos::parallel_reduce("srfEmiss_reduction_loop", N, KOKKOS_LAMBDA - (const int& i, double& lsum) { for(int i = 0; i < data_beg.data.nsectors; ++i) - { Kokkos::single(Kokkos::PerTeam(team), [&] { accum[icol] = accum[icol] + - linear_interp(data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), - delta_t_fraction); - - }); - } - data_out.emiss_sectors[0](icol) = accum[icol]; + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int icol = team.league_rank(); // column index + Real accum = 0; + // Parallel reduction over sectors + // FIXME: Do we need to use Kokkos::Single for each team here??? + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, nsectors), + [&](const int i, Real &update) { + const auto beg = data_beg.data.emiss_sectors[i](icol); + const auto end = data_end.data.emiss_sectors[i](icol); + update += linear_interp(beg, end, delta_t_fraction); + }, + accum); + // Assign the accumulated value to the output + data_out.emiss_sectors[0](icol) = accum; }); - Kokkos::fence();*/ } // perform_time_interpolation template @@ -332,32 +255,6 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate -template -template -void srfEmissFunctions::init_srf_emiss_objects( - const int ncol, const std::shared_ptr &grid, - const std::string &data_file, - const std::array §ors, - const std::string &srf_map_file, - // output - std::shared_ptr &SrfEmissHorizInterp, - srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, - srfEmissOutput &SrfEmissData_out, - std::shared_ptr &SrfEmissDataReader) { - // Init horizontal remap - SrfEmissHorizInterp = - create_horiz_remapper(grid, data_file, sectors, srf_map_file); - - // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, numSectors); - SrfEmissData_end = srfEmissInput(ncol, numSectors); - SrfEmissData_out.init(ncol, 1, true); - - // Create reader (an AtmosphereInput object) - SrfEmissDataReader = - create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); -} // init_srf_emiss_objects - template void srfEmissFunctions::init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, From 0c258d47649c18ffc4e30b8c2ab00c6f5b4744aa Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 12:25:53 -0700 Subject: [PATCH 318/477] Adds output fileds to yaml and also change the location of emissions files --- .../src/physics/mam/srf_emission_impl.hpp | 3 +- .../mam/constituent_fluxes/output.yaml | 33 ++++++++++++++++++- .../mam/emissions/CMakeLists.txt | 16 +++++++++ .../single-process/mam/emissions/input.yaml | 18 +++++----- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index ef99879277eb..5191966f588f 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -144,6 +144,7 @@ void srfEmissFunctions::perform_time_interpolation( // Assign the accumulated value to the output data_out.emiss_sectors[0](icol) = accum; }); + Kokkos::fence(); } // perform_time_interpolation template @@ -282,4 +283,4 @@ void srfEmissFunctions::init_srf_emiss_objects( } // namespace } // namespace scream::mam_coupling -#endif // SRF_EMISSION_IMPL_HPP \ No newline at end of file +#endif // SRF_EMISSION_IMPL_HPP diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml index e51b96eb5ce3..b237056b5cc1 100644 --- a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml @@ -5,7 +5,38 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + output_control: Frequency: 1 frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index b57adcb60ef0..3d64bc5b2b27 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -25,6 +25,22 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 2e63314ebb1e..06be9ec8a883 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,15 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From c29bab538696ea6fedbfac421d55a1049f4b6029 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 3 Aug 2024 13:42:55 -0700 Subject: [PATCH 319/477] Some GPU fixes but needs some cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 5c5129d1b7c1..564fa9465fae 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -266,6 +266,7 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -286,7 +287,8 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::fence(); // policy - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] @@ -296,7 +298,17 @@ void MAMConstituentFluxes::run_impl(const double dt) { nlev_, // output rpdel_); - std::cout << "BEFORE:::" << wet_aero_.gas_mmr[4](1, 1) << std::endl; + auto wet_aero = wet_aero_; + auto dry_atm = dry_atm_; + auto nlev = nlev_; + auto rpdel = rpdel_; + auto constituent_fluxes = constituent_fluxes_; + + + //auto aa = wet_aero.gas_mmr[4](1, 1); + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); + + printf("BEFORE:::%e\n",host_view(0,0)); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -307,14 +319,15 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get prognostics mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero_, icol); + mam_coupling::aerosols_for_column(wet_aero, icol); + //printf("in-loop-wetaero:,%e,%e\n", wet_aero.gas_mmr[4](0,0), progs_at_col.q_gas[4](0)); // get atmospheric quantities - haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); + haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); // Construct state_q (interstitial) array Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev_), [&](int klev) { + Kokkos::TeamVectorRange(team, nlev), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -323,25 +336,35 @@ void MAMConstituentFluxes::run_impl(const double dt) { mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); + // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); // Factor to convert units from kg/m2/s to kg/kg - Real unit_factor = dt * gravit * rpdel_(icol, klev); + Real unit_factor = dt * gravit * rpdel(icol, klev); // Update state with the constituent fluxes (modified // units) + printf("in-loop-klev:,%e\n", progs_at_col.q_gas[4](icol)); for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { state_q_at_lev_col[icnst] += - constituent_fluxes_(icol, icnst) * unit_factor; + constituent_fluxes(icol, icnst) * unit_factor; + //printf("in-loop-update:,%e,%e, %i\n", progs_at_col.q_gas[icnst](icol), state_q_at_lev_col[icnst], icnst); + printf("INLOOP:::%e, %i\n",state_q_at_lev_col[icnst], icnst); } mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, progs_at_col, klev); + }); }); - - std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; + auto aa1 = wet_aero.gas_mmr; + auto bb1 = aa1[4]; + auto host_view1 = Kokkos::create_mirror_view(bb1); + auto cc1 = host_view1(1,1); + printf("AFTER:::%e\n",cc1); + printf("Scientific notation:");//, wet_aero.gas_mmr[4](1, 1)); + //std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; } // run_impl ends // ============================================================================= From e4b0aa27768e0e760e7c8c8dc47ecb32c093581e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 3 Aug 2024 23:17:38 -0700 Subject: [PATCH 320/477] Reworked logic to add fluxes only at the surface level-needs cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 90 ++++++------------- ...eamxx_mam_constituent_fluxes_interface.hpp | 3 - 2 files changed, 29 insertions(+), 64 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 564fa9465fae..7fc360d92c88 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -249,12 +249,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(gas_mmr_field_name).get_view(); } - //----------------------------------------------------------------- - // Allocate memory - //----------------------------------------------------------------- - // Inverse of pseudo_density (1/Pa or or N/m2 or kg/m/s2) - rpdel_ = view_2d("rpdel_", ncol_, nlev_); - //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- @@ -266,7 +260,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -287,84 +280,59 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::fence(); // policy - //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] - // compute inverse of pseudo_density - mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, - nlev_, - // output - rpdel_); - auto wet_aero = wet_aero_; - auto dry_atm = dry_atm_; - auto nlev = nlev_; - auto rpdel = rpdel_; + auto wet_aero = wet_aero_; + auto dry_atm = dry_atm_; + const int surface_lev = nlev_ - 1; auto constituent_fluxes = constituent_fluxes_; + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); - - //auto aa = wet_aero.gas_mmr[4](1, 1); + // auto aa = wet_aero.gas_mmr[4](1, 1); auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); - printf("BEFORE:::%e\n",host_view(0,0)); + for(int icnst = 0; icnst < 5; ++icnst) { + printf("BEFORE:::%e,%e\n", host_view(icnst, 71), + wet_aero_.gas_mmr[4](icnst, 71)); + } + + const auto policy_pcnst = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol_, mam4::aero_model::pcnst); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // To construct state_q, we need fields from Prognostics (aerosols) - // and Atmosphere (water species such as qv, qc etc.) + EKAT_REQUIRE_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must not be zero!"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - // get prognostics mam4::Prognostics progs_at_col = mam_coupling::aerosols_for_column(wet_aero, icol); - //printf("in-loop-wetaero:,%e,%e\n", wet_aero.gas_mmr[4](0,0), progs_at_col.q_gas[4](0)); // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); + Real state_q_at_surf_lev[mam4::aero_model::pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); - // Construct state_q (interstitial) array - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_lev_col, klev); - - - // get the start index for gas species in the state_q array - int istart = mam4::utils::gasses_start_ind(); + const Real unit_factor = dt * gravit * rpdel; - // Factor to convert units from kg/m2/s to kg/kg - Real unit_factor = dt * gravit * rpdel(icol, klev); - - // Update state with the constituent fluxes (modified - // units) - printf("in-loop-klev:,%e\n", progs_at_col.q_gas[4](icol)); - for(int icnst = istart; icnst < mam4::aero_model::pcnst; - ++icnst) { - state_q_at_lev_col[icnst] += + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, istart, mam4::aero_model::pcnst), + [&](int icnst) { + state_q_at_surf_lev[icnst] += constituent_fluxes(icol, icnst) * unit_factor; - //printf("in-loop-update:,%e,%e, %i\n", progs_at_col.q_gas[icnst](icol), state_q_at_lev_col[icnst], icnst); - printf("INLOOP:::%e, %i\n",state_q_at_lev_col[icnst], icnst); - } - mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, - progs_at_col, klev); - }); + mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, + progs_at_col, surface_lev); }); - auto aa1 = wet_aero.gas_mmr; - auto bb1 = aa1[4]; - auto host_view1 = Kokkos::create_mirror_view(bb1); - auto cc1 = host_view1(1,1); - printf("AFTER:::%e\n",cc1); - printf("Scientific notation:");//, wet_aero.gas_mmr[4](1, 1)); - //std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; } // run_impl ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 448ae5464b87..f5d153360b96 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -15,7 +15,6 @@ namespace scream { // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMConstituentFluxes final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; - using view_2d = Field::view_dev_t; using const_view_2d = Field::view_dev_t; // number of horizontal columns @@ -36,8 +35,6 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { const_view_2d constituent_fluxes_; - view_2d rpdel_; // Inverse of pdel_ or pseudo_density - public: // Constructor MAMConstituentFluxes(const ekat::Comm &comm, From 4c5f58c63b3fecd2059bf5e0535b6e60e4646bb0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 4 Aug 2024 00:06:51 -0700 Subject: [PATCH 321/477] Fixes pm-gpu build and run-needs cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 7fc360d92c88..9fc712ba46f7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -285,10 +285,11 @@ void MAMConstituentFluxes::run_impl(const double dt) { using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] - + const int ncol = ncol_; auto wet_aero = wet_aero_; auto dry_atm = dry_atm_; const int surface_lev = nlev_ - 1; + static constexpr int pcnst = mam4::aero_model::pcnst; auto constituent_fluxes = constituent_fluxes_; // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); @@ -297,20 +298,19 @@ void MAMConstituentFluxes::run_impl(const double dt) { auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); for(int icnst = 0; icnst < 5; ++icnst) { - printf("BEFORE:::%e,%e\n", host_view(icnst, 71), - wet_aero_.gas_mmr[4](icnst, 71)); - } + printf("BEFORE:::%e\n", host_view(icnst, 71)); + } const auto policy_pcnst = ekat::ExeSpaceUtils::get_default_team_policy( - ncol_, mam4::aero_model::pcnst); + ncol, pcnst); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - EKAT_REQUIRE_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must not be zero!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must not be zero!\n"); const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); mam4::Prognostics progs_at_col = @@ -318,21 +318,24 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); - Real state_q_at_surf_lev[mam4::aero_model::pcnst] = {}; + Real state_q_at_surf_lev[pcnst] = {}; mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); const Real unit_factor = dt * gravit * rpdel; - + auto pcnst1 = pcnst; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, mam4::aero_model::pcnst), + Kokkos::TeamVectorRange(team, istart, pcnst1), [&](int icnst) { state_q_at_surf_lev[icnst] += constituent_fluxes(icol, icnst) * unit_factor; }); mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, - progs_at_col, surface_lev); + progs_at_col, surface_lev); }); + for(int icnst = 0; icnst < 5; ++icnst) { + printf("AFTER:::%e\n", host_view(icnst, 71)); + } } // run_impl ends // ============================================================================= From 209ba3db2733081f2125559c56359f6a989e8d5d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 5 Aug 2024 19:13:22 -0700 Subject: [PATCH 322/477] Moved Kokkos - loop outside of interface and verified that gas and aerosols mmrs are modified --- ...eamxx_mam_constituent_fluxes_functions.hpp | 105 ++++++++++++++++++ ...eamxx_mam_constituent_fluxes_interface.cpp | 72 +++--------- ...eamxx_mam_constituent_fluxes_interface.hpp | 2 + 3 files changed, 122 insertions(+), 57 deletions(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp new file mode 100644 index 000000000000..9059cf5ccbc6 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -0,0 +1,105 @@ +#ifndef EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP +#define EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP + +#include +#include + +namespace scream { + +namespace { + +void update_gas_aerosols_using_constituents( + const int ncol, const int nlev, const double dt, + const mam_coupling::DryAtmosphere &dry_atm, + const MAMConstituentFluxes::const_view_2d &constituent_fluxes, + // output + const mam_coupling::AerosolState &wet_aero) { + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr int pcnst = mam4::aero_model::pcnst; + + // Declare local variables + const int ncol_loc = ncol; + auto wet_aero_loc = wet_aero; + auto dry_atm_loc = dry_atm; + const int surface_lev = nlev - 1; + auto constituent_fluxes_loc = constituent_fluxes; + + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); + + // number of constituents to update + const int nconstituents = pcnst - istart; + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, surface_lev), icnst + 9); + } + + // Create a policy to loop over columns annd number of constituents + // to update + // const auto policy = + // ekat::ExeSpaceUtils::get_default_team_policy(ncol, + // nconstituents); + const auto policy = ekat::ExeSpaceUtils:: + get_default_team_policy(1, nconstituents); + + // Loop through all columns to update tracer mixing rations + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + //---------------------------------------------------------------------- + // To form EAM like state%q array, we need prognostics (gas and aerosol + // mmrs) atmosphere (qv, qc, nc, ni, etc.) + //---------------------------------------------------------------------- + + // Get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(wet_aero, // output + icol); // input + + // Get atmospheric quantities + const haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atm, // output + icol); // input + + // Form state%q like array + Real state_q_at_surf_lev[pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, // input + state_q_at_surf_lev, // output + surface_lev); // input + + // Compute the units conversion factor (kg/m2/s to kg/kg) + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must be non-zero!\n"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); + const Real unit_factor = dt * gravit * rpdel; + + // Loop for + auto pcnst_loc = pcnst; + for(int icnst = 9; icnst < 15; ++icnst) { + printf("BEFORE-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], + progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); + } + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, istart, pcnst_loc), [&](int icnst) { + state_q_at_surf_lev[icnst] += + constituent_fluxes(icol, icnst) * unit_factor; + }); // pcsnt loop + mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, // input + progs_at_col, // output + surface_lev); // input + + for(int icnst = 9; icnst < 15; ++icnst) { + printf("After-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], + progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); + } + }); // icol loop +} + +} // namespace +} // namespace scream + +#endif // ifdef EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 9fc712ba46f7..1d1ffca29285 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -1,3 +1,4 @@ +#include #include namespace scream { @@ -260,8 +261,12 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { + // const auto scan_policy = ekat::ExeSpaceUtils< + // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, + // nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in @@ -279,63 +284,16 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // policy - // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view1 = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + printf("END:::%e, %i\n", host_view1(0, 71), icnst + 9); + } - using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - const int ncol = ncol_; - auto wet_aero = wet_aero_; - auto dry_atm = dry_atm_; - const int surface_lev = nlev_ - 1; - static constexpr int pcnst = mam4::aero_model::pcnst; - auto constituent_fluxes = constituent_fluxes_; - // get the start index for gas species in the state_q array - int istart = mam4::utils::gasses_start_ind(); - - // auto aa = wet_aero.gas_mmr[4](1, 1); - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); - - for(int icnst = 0; icnst < 5; ++icnst) { - printf("BEFORE:::%e\n", host_view(icnst, 71)); - } - - const auto policy_pcnst = - ekat::ExeSpaceUtils::get_default_team_policy( - ncol, pcnst); - // Loop through all columns to update tracer mixing rations - Kokkos::parallel_for( - policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must not be zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, icol); - - // get atmospheric quantities - haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); - - const Real unit_factor = dt * gravit * rpdel; - auto pcnst1 = pcnst; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, pcnst1), - [&](int icnst) { - state_q_at_surf_lev[icnst] += - constituent_fluxes(icol, icnst) * unit_factor; - }); - mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, - progs_at_col, surface_lev); - }); - for(int icnst = 0; icnst < 5; ++icnst) { - printf("AFTER:::%e\n", host_view(icnst, 71)); - } } // run_impl ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index f5d153360b96..7e77fccf2afc 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -14,9 +14,11 @@ namespace scream { // The process responsible for applying MAM4 constituent fluxes. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMConstituentFluxes final : public scream::AtmosphereProcess { + public: using KT = ekat::KokkosTypes; using const_view_2d = Field::view_dev_t; + private: // number of horizontal columns int ncol_, nlev_; From d4c9e2ffbcedb5fedb26911d578ac2b403dfd062 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 9 Aug 2024 08:06:45 -0700 Subject: [PATCH 323/477] Fixes some GPU errors and warnings --- ...eamxx_mam_constituent_fluxes_functions.hpp | 107 +++++++----------- ...eamxx_mam_constituent_fluxes_interface.cpp | 78 +++++++------ 2 files changed, 89 insertions(+), 96 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index 9059cf5ccbc6..c53c09eacbd9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -19,7 +19,6 @@ void update_gas_aerosols_using_constituents( static constexpr int pcnst = mam4::aero_model::pcnst; // Declare local variables - const int ncol_loc = ncol; auto wet_aero_loc = wet_aero; auto dry_atm_loc = dry_atm; const int surface_lev = nlev - 1; @@ -28,75 +27,57 @@ void update_gas_aerosols_using_constituents( // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - // number of constituents to update + // number of constituents to update (currently updating only MAM4xx constituents) const int nconstituents = pcnst - istart; - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, surface_lev), icnst + 9); - } - // Create a policy to loop over columns annd number of constituents // to update - // const auto policy = - // ekat::ExeSpaceUtils::get_default_team_policy(ncol, - // nconstituents); - const auto policy = ekat::ExeSpaceUtils:: - get_default_team_policy(1, nconstituents); + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol, + nconstituents); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - //---------------------------------------------------------------------- - // To form EAM like state%q array, we need prognostics (gas and aerosol - // mmrs) atmosphere (qv, qc, nc, ni, etc.) - //---------------------------------------------------------------------- - - // Get prognostics - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, // output - icol); // input - - // Get atmospheric quantities - const haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atm, // output - icol); // input - - // Form state%q like array - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, // input - state_q_at_surf_lev, // output - surface_lev); // input - - // Compute the units conversion factor (kg/m2/s to kg/kg) - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must be non-zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - const Real unit_factor = dt * gravit * rpdel; - - // Loop for - auto pcnst_loc = pcnst; - for(int icnst = 9; icnst < 15; ++icnst) { - printf("BEFORE-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], - progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); - } - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, pcnst_loc), [&](int icnst) { - state_q_at_surf_lev[icnst] += - constituent_fluxes(icol, icnst) * unit_factor; - }); // pcsnt loop - mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, // input - progs_at_col, // output - surface_lev); // input - - for(int icnst = 9; icnst < 15; ++icnst) { - printf("After-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], - progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); - } - }); // icol loop + policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + //---------------------------------------------------------------------- + // To form EAM like state%q array, we need prognostics (gas and aerosol + // mmrs) atmosphere (qv, qc, nc, ni, etc.) + //---------------------------------------------------------------------- + + // Get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(wet_aero, // output + icol); // input + // Get atmospheric quantities + const haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atm, // output + icol); // input + + // Form state%q like array + Real state_q_at_surf_lev[pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, // input + state_q_at_surf_lev, // output + surface_lev); // input + + // Compute the units conversion factor (kg/m2/s to kg/kg) + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must be non-zero!\n"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); + const Real unit_factor = dt * gravit * rpdel; + + // Update state vector with constituent fluxes + for (int icnst = istart; icnst #include #include namespace scream { @@ -260,39 +261,50 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // ================================================================ // RUN_IMPL // ================================================================ -void MAMConstituentFluxes::run_impl(const double dt) { - // const auto scan_policy = ekat::ExeSpaceUtils< - // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, - // nlev_); - - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); - - // ------------------------------------------------------------------- - // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in - // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". - // We are converting wet atm to dry atm. Since we do not use or update - // any of the water constituents (qc, qv, qi etc.), we should be okay - // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. - // ------------------------------------------------------------------- - - // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, - constituent_fluxes_, - // output - wet_aero_); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view1 = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - printf("END:::%e, %i\n", host_view1(0, 71), icnst + 9); - } + void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, + nlev_); + + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + + // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ etc. + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } + auto start = std::chrono::steady_clock::now(); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + auto stop = std::chrono::steady_clock::now(); + auto duration = (stop - start); + + + // To get the value of duration use the count() + // member function on the duration object + printf("TIME:%e\n", std::chrono::duration(duration).count()); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } } // run_impl ends From 6f1e5d72d5f9e2135f18cd01e5042bc2d1e4bfd4 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Sat, 10 Aug 2024 07:51:55 -0600 Subject: [PATCH 324/477] Fix some compile warnings that are really errors. Calling host from device is not allowed and will fail at runtime. Also there were a couple of warnings about the implicit capture of the this pointer that turned out to cause runtime errors. --- ...and_online_emissions_process_interface.cpp | 36 ++++++------------- ...and_online_emissions_process_interface.hpp | 16 --------- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- .../eamxx/src/physics/mam/srf_emission.hpp | 27 +++++++------- .../src/physics/mam/srf_emission_impl.hpp | 21 +++++------ 5 files changed, 35 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 91a5b161dff4..09c173468a2f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -191,7 +191,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- // Work array to store fluxes after unit conversions to kg/m2/s - fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); + fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units", ncol_); // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; @@ -221,16 +221,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, 1); - // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - // policy to loop over columns only - const auto ncol_only_policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol_, 1); + // Zero output + Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -261,25 +254,18 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { const int species_index = spcIndex_in_pcnst_.at(ispec_srf.species_name); // modify units from molecules/cm2/s to kg/m2/s + auto fluxes_in_mks_units = this->fluxes_in_mks_units_; + auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units - Kokkos::parallel_for( - ncol_only_policy, - KOKKOS_LAMBDA(const MAMSrfOnlineEmiss::KT::MemberType &team) { - const int icol = team.league_rank(); - fluxes_in_mks_units_(icol) = - ispec_srf.data_out_.emiss_sectors[0](icol) * mfactor; - }); - - // Get subview - auto flux_1d_view = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - spcIndex_in_pcnst_.at(ispec_srf.species_name)); - - // update flux in constituent_fluxes_ view - Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); + Kokkos::parallel_for("fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = ispec_srf.data_out_.emiss_sectors(0,icol) * mfactor; + constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); + }); + } // for loop for species + Kokkos::fence(); } // run_imple ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index b67b1de0bd7d..031fb62d8b75 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -83,24 +83,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { void initialize(const view_2d &constituent_fluxes) { constituent_fluxes_pre_ = constituent_fluxes; } - - KOKKOS_INLINE_FUNCTION - void operator()( - const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index - // zero-out the constituent surface fluxes for all gas and aerosol - // species. - for(auto icnst = mam4::utils::gasses_start_ind(); - icnst < mam4::aero_model::pcnst; ++icnst) { - constituent_fluxes_pre_(i, icnst) = 0; - } - team.team_barrier(); - - } // operator() - // local variables for preprocess struct view_2d constituent_fluxes_pre_; - }; // MAMSrfOnlineEmiss::Preprocess private: diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 09d64d6028d9..a3a5f746aa30 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -777,7 +777,7 @@ void compute_wet_mixing_ratios(const Team& team, } // Computes the reciprocal of pseudo density for a column -KOKKOS_INLINE_FUNCTION +inline void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, const_view_2d pdel, const int nlev, diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 82a4b3113821..886d9d00609f 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -30,26 +30,23 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_, const int nsectors_) { - init(ncol_, nsectors_, true); + srfEmissData(const int ncol_, const int nsectors_) : + ncols(ncol_), + nsectors(nsectors_) + { + init(ncols, nsectors, true); } - void init(const int ncol_, const int nsectors_, const bool allocate) { - ncols = ncol_; - nsectors = nsectors_; - - if(allocate) { - for(int i = 0; i < nsectors; ++i) { - auto sector = view_1d("", ncols); - emiss_sectors.push_back(sector); - } - } + void init(const int ncol, const int nsector, const bool allocate) { + ncols = ncol; + nsectors = nsector; + if(allocate) + emiss_sectors = view_2d("AllSectors", nsectors, ncols); } // srfEmissData init // Basic spatial dimensions of the data int ncols, nsectors; - std::vector emiss_sectors; - + view_2d emiss_sectors; }; // srfEmissData struct srfEmissInput { @@ -141,4 +138,4 @@ struct srfEmissFunctions { } // namespace scream::mam_coupling #endif // SRF_EMISSION_HPP -#include "srf_emission_impl.hpp" \ No newline at end of file +#include "srf_emission_impl.hpp" diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 5191966f588f..35a2b90deeb0 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -55,14 +55,14 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - std::vector emiss_sectors; + std::vector field_emiss_sectors; for(int icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); - emiss_sectors.push_back(f); + field_emiss_sectors.push_back(f); remapper->register_field_from_tgt(f); } @@ -76,18 +76,18 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector emiss_sectors; + std::vector field_emiss_sectors; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - emiss_sectors.push_back(horiz_remapper->get_src_field(i)); + field_emiss_sectors.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - emiss_sectors, true); + field_emiss_sectors, true); } // create_srfEmiss_data_reader template template -KOKKOS_INLINE_FUNCTION ScalarX srfEmissFunctions::linear_interp( +ScalarX srfEmissFunctions::linear_interp( const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { return (1 - t) * x0 + t * x1; } // linear_interp @@ -136,13 +136,13 @@ void srfEmissFunctions::perform_time_interpolation( Kokkos::parallel_reduce( Kokkos::TeamThreadRange(team, nsectors), [&](const int i, Real &update) { - const auto beg = data_beg.data.emiss_sectors[i](icol); - const auto end = data_end.data.emiss_sectors[i](icol); + const auto beg = data_beg.data.emiss_sectors(i, icol); + const auto end = data_end.data.emiss_sectors(i, icol); update += linear_interp(beg, end, delta_t_fraction); }, accum); // Assign the accumulated value to the output - data_out.emiss_sectors[0](icol) = accum; + data_out.emiss_sectors(0, icol) = accum; }); Kokkos::fence(); } // perform_time_interpolation @@ -212,7 +212,8 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], sector); + const auto emiss = Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); + Kokkos::deep_copy(emiss, sector); } Kokkos::fence(); From 9c2fb6545c13b86ba1583be90bc36f71f13efd68 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 11:54:38 -0700 Subject: [PATCH 325/477] Fixes a warning for a size type for a string vector --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 35a2b90deeb0..7a677bbdca3e 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,7 +57,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(int icomp = 0; icomp < sector_names.size(); ++icomp) { + for(std::vector::size_type icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From c45a86a5d267d1a460990ca05b38522fa4662006 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 12:02:47 -0700 Subject: [PATCH 326/477] Clang format --- ...eamxx_mam_constituent_fluxes_functions.hpp | 87 +++++++++--------- ...eamxx_mam_constituent_fluxes_interface.cpp | 88 +++++++++---------- ...and_online_emissions_process_interface.cpp | 15 ++-- .../eamxx/src/physics/mam/srf_emission.hpp | 11 +-- .../src/physics/mam/srf_emission_impl.hpp | 11 ++- 5 files changed, 106 insertions(+), 106 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index c53c09eacbd9..e7ffb6376c35 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -27,57 +27,56 @@ void update_gas_aerosols_using_constituents( // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - // number of constituents to update (currently updating only MAM4xx constituents) + // number of constituents to update (currently updating only MAM4xx + // constituents) const int nconstituents = pcnst - istart; // Create a policy to loop over columns annd number of constituents // to update - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol, - nconstituents); + const auto policy = ekat::ExeSpaceUtils:: + get_default_team_policy(ncol, nconstituents); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - //---------------------------------------------------------------------- - // To form EAM like state%q array, we need prognostics (gas and aerosol - // mmrs) atmosphere (qv, qc, nc, ni, etc.) - //---------------------------------------------------------------------- - - // Get prognostics - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, // output - icol); // input - // Get atmospheric quantities - const haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atm, // output - icol); // input - - // Form state%q like array - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, // input - state_q_at_surf_lev, // output - surface_lev); // input - - // Compute the units conversion factor (kg/m2/s to kg/kg) - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must be non-zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - const Real unit_factor = dt * gravit * rpdel; - - // Update state vector with constituent fluxes - for (int icnst = istart; icnst::get_thread_range_parallel_scan_team_policy(ncol_, - nlev_); - - // ------------------------------------------------------------------- - // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in - // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". - // We are converting wet atm to dry atm. Since we do not use or update - // any of the water constituents (qc, qv, qi etc.), we should be okay - // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. - // ------------------------------------------------------------------- - - // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ etc. - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - auto start = std::chrono::steady_clock::now(); - update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, - constituent_fluxes_, - // output - wet_aero_); - auto stop = std::chrono::steady_clock::now(); - auto duration = (stop - start); - - - // To get the value of duration use the count() - // member function on the duration object - printf("TIME:%e\n", std::chrono::duration(duration).count()); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } +void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + + // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ + // etc. + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } + auto start = std::chrono::steady_clock::now(); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + auto stop = std::chrono::steady_clock::now(); + auto duration = (stop - start); + + // To get the value of duration use the count() + // member function on the duration object + printf("TIME:%e\n", + std::chrono::duration(duration).count()); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } } // run_impl ends diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 09c173468a2f..a4ac4778982d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -221,8 +221,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - - // Zero output + // Zero output Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation @@ -255,14 +254,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // modify units from molecules/cm2/s to kg/m2/s auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - auto constituent_fluxes = this->constituent_fluxes_; + auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units - Kokkos::parallel_for("fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - fluxes_in_mks_units(icol) = ispec_srf.data_out_.emiss_sectors(0,icol) * mfactor; - constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); - }); + Kokkos::parallel_for( + "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = + ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; + constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); + }); } // for loop for species Kokkos::fence(); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 886d9d00609f..29aaca421ea9 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -30,18 +30,15 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_, const int nsectors_) : - ncols(ncol_), - nsectors(nsectors_) - { + srfEmissData(const int ncol_, const int nsectors_) + : ncols(ncol_), nsectors(nsectors_) { init(ncols, nsectors, true); } void init(const int ncol, const int nsector, const bool allocate) { - ncols = ncol; + ncols = ncol; nsectors = nsector; - if(allocate) - emiss_sectors = view_2d("AllSectors", nsectors, ncols); + if(allocate) emiss_sectors = view_2d("AllSectors", nsectors, ncols); } // srfEmissData init // Basic spatial dimensions of the data diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 7a677bbdca3e..dfe1190cdb91 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,7 +57,8 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(std::vector::size_type icomp = 0; icomp < sector_names.size(); ++icomp) { + for(std::vector::size_type icomp = 0; + icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); @@ -87,8 +88,9 @@ srfEmissFunctions::create_srfEmiss_data_reader( template template -ScalarX srfEmissFunctions::linear_interp( - const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { +ScalarX srfEmissFunctions::linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t) { return (1 - t) * x0 + t * x1; } // linear_interp @@ -212,7 +214,8 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - const auto emiss = Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); + const auto emiss = + Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); Kokkos::deep_copy(emiss, sector); } From 245571a6e73fba2f3539e49c8442a767a874f61b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:33:00 -0700 Subject: [PATCH 327/477] Store size in local in to avoid declaring long size_type datatype --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index dfe1190cdb91..7b5195c69432 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,8 +57,9 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(std::vector::size_type icomp = 0; - icomp < sector_names.size(); ++icomp) { + const int sector_size = sector_names.size(); + for(int icomp = 0; + icomp < sector_size; ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From 029ba9ccf009550d017f67f69c9e931393d3ef4e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:34:42 -0700 Subject: [PATCH 328/477] clang format --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 7b5195c69432..2d29075e41a4 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -58,8 +58,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; const int sector_size = sector_names.size(); - for(int icomp = 0; - icomp < sector_size; ++icomp) { + for(int icomp = 0; icomp < sector_size; ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From a816871c21f9550799d751cd69e6ed1038dec076 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:40:54 -0700 Subject: [PATCH 329/477] Revert "Merge branch 'master' into singhbalwinder/branch_mjs/mam4xx/emissionsIntegration" This reverts commit c2f7d298d93a06ad3c75a4d42f5139d08c7ed528, reversing changes made to 2d481948a48c0d9e006a0c2038328ca3642c499d. --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- cime_config/machines/config_machines.xml | 51 ++-- .../cmake/machine-files/anlgce-ub22.cmake | 12 - .../eamxx/cmake/machine-files/anlgce.cmake | 6 +- components/eamxx/scripts/machines_specs.py | 4 - .../src/dynamics/homme/eamxx_homme_iop.cpp | 259 +++++++++--------- ...mxx_mam_microphysics_process_interface.cpp | 4 +- components/eamxx/src/physics/p3/p3_iso_c.f90 | 19 +- .../atm_process/atmosphere_process_hash.cpp | 4 +- .../eamxx/src/share/io/scorpio_output.cpp | 35 +-- .../iop/intensive_observation_period.cpp | 12 +- 13 files changed, 181 insertions(+), 231 deletions(-) delete mode 100644 components/eamxx/cmake/machine-files/anlgce-ub22.cmake diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 9ea25ae1864e..ccca0c479f26 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index e6fb53ba29f5..abd8b92e4f6c 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index e0a8e19f9c01..5ecdf6dec00c 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.11" - name: Run unit tests diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index eeb80756bb8c..6765d000e26c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -368,7 +368,6 @@ nvidia aocc cudatoolkit - cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -379,24 +378,24 @@ - PrgEnv-gnu/8.5.0 - gcc-native/12.3 + PrgEnv-gnu/8.3.3 + gcc/11.2.0 PrgEnv-nvidia - nvidia/24.5 + nvidia/22.7 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - gcc-native-mixed/12.3 + gcc-mixed/11.2.0 @@ -408,12 +407,12 @@ - cray-libsci/23.12.5 - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 + cray-libsci/23.02.1.1 + craype/2.7.20 + cray-mpich/8.1.25 + cray-hdf5-parallel/1.12.2.3 + cray-netcdf-hdf5parallel/4.9.0.3 + cray-parallel-netcdf/1.12.3.3 cmake/3.24.3 @@ -678,7 +677,6 @@ nvidia aocc cudatoolkit - cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -689,24 +687,26 @@ - PrgEnv-gnu/8.5.0 - gcc-native/12.3 + PrgEnv-gnu/8.3.3 + gcc/11.2.0 + PrgEnv-nvidia - nvidia/24.5 + nvidia/23.9 - cudatoolkit/12.2 + cudatoolkit/11.7 + craype-accel-nvidia80 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - gcc-native-mixed/12.3 @@ -718,13 +718,20 @@ - cray-libsci/23.12.5 + cray-libsci/23.02.1.1 + craype/2.7.20 + cray-mpich/8.1.25 + cray-hdf5-parallel/1.12.2.3 + cray-netcdf-hdf5parallel/4.9.0.3 + cray-parallel-netcdf/1.12.3.3 + cmake/3.24.3 + evp-patch diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake deleted file mode 100644 index e79f70015471..000000000000 --- a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake +++ /dev/null @@ -1,12 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -# Remove this if you are using a resource manager (slurm etc) -set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") - -# EKAT MPI settings -set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") -set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index e79f70015471..079000059d19 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) + # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 43e5ebfa083c..9536d415c0de 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,10 +82,6 @@ ["mpicxx","mpifort","mpicc"], "", ""), - "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], - ["mpicxx","mpifort","mpicc"], - "", - ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 74579df552bd..eaf65a69cda5 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,14 +10,14 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" +#include "ElementOps.hpp" +#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" +#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" -// SCREAM includes -#include "share/util/scream_common_physics_functions.hpp" - // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -222,7 +222,11 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - using PF = PhysicsFunctions; + + using EOS = Homme::EquationOfState; + using ElementOps = Homme::ElementOps; + using KV = Homme::KernelVariables; + using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -259,7 +263,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states + // Homme element states and EOS/EO classes auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -268,6 +272,11 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); + EOS eos; + eos.init(params.theta_hydrostatic_mode, hvcoord); + + ElementOps elem_ops; + elem_ops.init(hvcoord); const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); // Load data from IOP files, if necessary @@ -310,66 +319,92 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for eamxx - const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + // Team policy and workspace manager for both homme and scream + // related loops. We need separate policies since hommexx functions used here + // assume they are called inside nested loops for elements and Gaussian points, + // whereas EAMxx function we use expects a single level of parallelism + // for elements and Guassian points. + // TODO: scream::ColumnOps functions could take an arbitary loop boundary + // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that + // all 3 kernel launches here could be combined. + const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); + const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); + WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); + WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); view_Nd + rstar ("rstar", nelem, NGP, NGP, NLEV), + exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing temperature - auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; + // Lambda for computing rstar, exner, and temperature from Hommexx + auto compute_homme_states = [&] () { + Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); + auto ws = homme_wsm.get_workspace(team); + auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); + uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto pnh_i = ekat::subview(pnh, idx); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); + Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); + Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); + Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); + + // Compute exner from EOS + if (theta_hydrostatic_mode) { + auto hydro_p_int = ws.take("hydro_p_int"); + Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); + elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); + eos.compute_exner(kv, pnh_scalar, exner_scalar); + ws.release(hydro_p_int); + } else { + eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); + } + + // Get the temperature from dynamics states + elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); }); // Release WS views - ws.release_many_contiguous<1>({&pmid}); + ws.release_macro_block(pnh_slot, NGP*NGP); }); }; - // Preprocess some homme states to get temperature - compute_temperature(); + // Preprocess some homme states to get temperature and exner + compute_homme_states(); Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); + auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -414,57 +449,44 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - team.team_barrier(); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + // Recompute rstar with updated qv, and convert updated temperature back to potential temperature + elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + }); }); - - // Release WS views - ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); if (iop_nudge_tq or iop_nudge_uv) { @@ -472,8 +494,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute temperature - compute_temperature(); + // Compute rstar, exner and temperature from Hommexx + compute_homme_states(); Kokkos::fence(); } @@ -549,36 +571,25 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_iop, + policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - if (iop_nudge_tq or iop_nudge_uv) { - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { + KV kv(team); + const int ie = team.league_rank(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high @@ -588,26 +599,22 @@ apply_iop_forcing(const Real dt) for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + and + pressure_from_iop >= iop_nudge_tq_high*100); } qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + // Convert updated temperature back to potential temperature + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); } }); - } - - // Release WS views - ws.release_many_contiguous<1>({&pmid}); + }); }); } } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 8b93936148ce..7ea2adfddb63 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,8 +317,6 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs - auto some_step = step_; - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index @@ -450,7 +448,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config.amicphys, some_step, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index ea0a18411c10..d75af1cb2edb 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=512), intent(out) :: string + character(kind=c_char, len=256), intent(inout) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,17 +69,10 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. -#ifdef SCREAM_DOUBLE_PRECISION - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR -#else - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR -#endif + call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") + call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") + call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") + call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") if (write_tables) then call p3_init_b() diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 37cb251d7796..9fb5726084f4 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index a105e244d76f..4cb3f50403ac 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,42 +1038,11 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { - std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - if (mode==scorpio::FileMode::Append) { - // Similar to the regular fields above, check that the var is in the file, and has the right properties - EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), - "Error! Cannot append, due to variable missing from the file.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n"); - const auto& var = scorpio::get_var(filename,name); - EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, - "Error! Cannot append, due to variable dimensions mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var dims : " + ekat::join(vec_of_dims,",") + "\n" - " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); - EKAT_REQUIRE_MSG (var.units==unitless, - "Error! Cannot append, due to variable units mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var units: " + unitless + "\n" - " - var units from file: " + var.units + "\n"); - EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, - "Error! Cannot append, due to time dependency mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" - " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); - } else { - // Note, unlike with regular output variables, for the average counting - // variables we don't need to add all of the extra metadata. So we simply - // define the variable. - scorpio::define_var(filename, name, unitless, vec_of_dims, - "real",fp_precision, m_add_time_dim); - } + scorpio::define_var(filename, name, "unitless", vec_of_dims, + "real",fp_precision, m_add_time_dim); } } } // register_variables diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 32b120c57002..76e039e65ab5 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -270,9 +270,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),(Real)0.1); + m_params.get("target_latitude"); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - std::max(m_params.get("target_longitude"),(Real)0.1); + m_params.get("target_longitude"); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -543,14 +543,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); - // Read in IOP lev data - auto data = iop_file_pressure.get_view().data(); - scorpio::read_var(iop_file,"lev",data); - - // Convert to pressure to millibar (file gives pressure in Pa) - for (int ilev=0; ilev(); From a7e719109b41762b6c63eb572a06b10f08351f4c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 16:40:41 -0700 Subject: [PATCH 330/477] Rebases and update mam4 submodule to point to its latest master --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 5495c27a6df7..2eee1988b1e6 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d +Subproject commit 2eee1988b1e6488c904e0a28564bdcadfa190d34 From 20b3d5e223d31b136bee335664704bec0f0aaee7 Mon Sep 17 00:00:00 2001 From: jsbamboo Date: Tue, 13 Aug 2024 18:27:51 -0700 Subject: [PATCH 331/477] update Livermore Computing machine config for eamxx --- .../eamxx/cmake/machine-files/dane-intel.cmake | 6 ++++++ components/eamxx/cmake/machine-files/dane.cmake | 13 +++++++++++++ components/eamxx/cmake/machine-files/ruby.cmake | 2 -- components/eamxx/scripts/machines_specs.py | 12 ++++++++---- 4 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 components/eamxx/cmake/machine-files/dane-intel.cmake create mode 100644 components/eamxx/cmake/machine-files/dane.cmake diff --git a/components/eamxx/cmake/machine-files/dane-intel.cmake b/components/eamxx/cmake/machine-files/dane-intel.cmake new file mode 100644 index 000000000000..df6aad60256b --- /dev/null +++ b/components/eamxx/cmake/machine-files/dane-intel.cmake @@ -0,0 +1,6 @@ +include(${CMAKE_CURRENT_LIST_DIR}/dane.cmake) +set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64/ -qmkl" CACHE STRING "" FORCE) +set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) +set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) +execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/dane.cmake b/components/eamxx/cmake/machine-files/dane.cmake new file mode 100644 index 000000000000..aec9f2b42227 --- /dev/null +++ b/components/eamxx/cmake/machine-files/dane.cmake @@ -0,0 +1,13 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Load all kokkos settings from Ekat's mach file +set(Kokkos_ENABLE_SERIAL TRUE CACHE BOOL "") + +# Enable Broadwell arch in Kokkos +option(Kokkos_ARCH_BDW "" ON) +# Load Broadwell flags and openmp backend for kokkos +include (${EKAT_MACH_FILES_PATH}/kokkos/intel-bdw.cmake) +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) + +include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) diff --git a/components/eamxx/cmake/machine-files/ruby.cmake b/components/eamxx/cmake/machine-files/ruby.cmake index 77d6e6618c71..aec9f2b42227 100644 --- a/components/eamxx/cmake/machine-files/ruby.cmake +++ b/components/eamxx/cmake/machine-files/ruby.cmake @@ -11,5 +11,3 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/intel-bdw.cmake) include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) - -set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 43e5ebfa083c..337f8b66d222 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -36,17 +36,21 @@ "", "/sems-data-store/ACME/baselines/scream/master-baselines"), "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'", - "export PATH=/usr/gdata/e3sm/netcdf/bin:$PATH", - "export LD_LIBRARY_PATH=/usr/gdata/e3sm/netcdf/lib:$LD_LIBRARY_PATH", + "export PATH=/usr/workspace/e3sm/netcdf/bin:$PATH", + "export LD_LIBRARY_PATH=/usr/workspace/e3sm/netcdf/lib:$LD_LIBRARY_PATH", ], ["mpicxx","mpifort","mpicc"], "bsub -Ip -qpdebug", ""), - "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "ruby-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), - "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "dane-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + ["mpicxx","mpifort","mpicc"], + "salloc --partition=pdebug", + ""), + "quartz-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), From f1b2ab67274e7120808df834bc82ae7e38ebc9b3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 21:10:12 -0700 Subject: [PATCH 332/477] Adds a multi-process test with srf emiss and constituent fluxes --- .../mam4xx/srf_online_emiss/shell_commands | 13 ----- .../multi-process/physics_only/CMakeLists.txt | 1 + .../CMakeLists.txt | 53 +++++++++++++++++++ .../input.yaml | 45 ++++++++++++++++ .../output.yaml | 41 ++++++++++++++ 5 files changed, 140 insertions(+), 13 deletions(-) delete mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands deleted file mode 100644 index 6995cd8b8562..000000000000 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands +++ /dev/null @@ -1,13 +0,0 @@ - -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" - -#modify initial condition file to get aerosol species ICs -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b - -# Add spa as RRTMG needs spa -$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b - - - diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index d6ed13dfb0a7..bb23911ff1c2 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -11,6 +11,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) add_subdirectory(mam/p3_mam4_wetscav) add_subdirectory(mam/shoc_cldfrac_p3_wetscav) + add_subdirectory(mam/mam4_srf_online_emiss_mam4_constituent_fluxes) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt new file mode 100644 index 000000000000..8d2f2653387e --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt @@ -0,0 +1,53 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME mam4_srf_online_emiss_mam4_constituent_fluxes) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS mam + LABELS physics mam4_srf_online_emiss mam4_constituent_fluxes + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS physics mam4_srf_online_emiss mam4_constituent_fluxes + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml new file mode 100644 index 000000000000..45fac36611e4 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -0,0 +1,45 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_srf_online_emiss, mam4_constituent_fluxes] + schedule_type: Sequential + mam4_srf_online_emiss: + # MAM4xx-Surface-Emissions + srf_remap_file: "" + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml new file mode 100644 index 000000000000..15e60e91517c --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml @@ -0,0 +1,41 @@ +%YAML 1.1 +--- +filename_prefix: shoc_mam4_drydep_output +Averaging Type: Instant +Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - constituent_fluxes + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 2bf266178d60a74c08ad005632e8b14c2de7f81b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 14 Aug 2024 14:06:41 -0700 Subject: [PATCH 333/477] Adds testmods and files for ne4pg2 CIME simulation --- .../cime_config/namelist_defaults_scream.xml | 29 ++++++++++--------- .../shell_commands | 17 +++++++++++ ...and_online_emissions_process_interface.cpp | 2 +- .../input.yaml | 2 ++ 4 files changed, 35 insertions(+), 15 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index e8e062cfaeb4..12095edff394 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -242,8 +242,7 @@ be lost if SCREAM_HACK_XML is not enabled. - - + @@ -266,22 +265,24 @@ be lost if SCREAM_HACK_XML is not enabled. - - + + - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240814.nc + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands new file mode 100644 index 000000000000..05b0b4954116 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -0,0 +1,17 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +#Update IC file and add the processes +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a4ac4778982d..f2a68ee6b5a7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -40,7 +40,7 @@ void MAMSrfOnlineEmiss::set_grids( grid_name); // Surface emissions remapping file - std::string srf_map_file = m_params.get("srf_remap_file"); + auto srf_map_file = m_params.get("srf_remap_file", ""); //-------------------------------------------------------------------- // Init dms srf emiss data structures diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 45fac36611e4..24c1d9f886e2 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -39,6 +39,8 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 0.0 + # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From 5879c571b60dc1ed89a78bdd3b2b4fb6f55daa97 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 14 Aug 2024 23:21:05 -0700 Subject: [PATCH 334/477] Revertes codes that were changed after the rebase --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- cime_config/machines/config_machines.xml | 51 ++-- .../cmake/machine-files/anlgce-ub22.cmake | 12 + .../eamxx/cmake/machine-files/anlgce.cmake | 6 +- components/eamxx/scripts/machines_specs.py | 4 + .../src/dynamics/homme/eamxx_homme_iop.cpp | 259 +++++++++--------- ...mxx_mam_microphysics_process_interface.cpp | 2 + components/eamxx/src/physics/p3/p3_iso_c.f90 | 19 +- .../atm_process/atmosphere_process_hash.cpp | 4 +- .../eamxx/src/share/io/scorpio_output.cpp | 35 ++- .../src/share/io/scream_scorpio_interface.cpp | 6 +- .../iop/intensive_observation_period.cpp | 12 +- .../single-process/mam/optics/CMakeLists.txt | 1 - 15 files changed, 233 insertions(+), 184 deletions(-) create mode 100644 components/eamxx/cmake/machine-files/anlgce-ub22.cmake diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ccca0c479f26..9ea25ae1864e 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index abd8b92e4f6c..e6fb53ba29f5 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 5ecdf6dec00c..e0a8e19f9c01 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" - name: Run unit tests diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6765d000e26c..eeb80756bb8c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -368,6 +368,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -378,24 +379,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/22.7 + nvidia/24.5 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - gcc-mixed/11.2.0 + gcc-native-mixed/12.3 @@ -407,12 +408,12 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + cray-libsci/23.12.5 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 @@ -677,6 +678,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -687,26 +689,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 - + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 - cudatoolkit/11.7 - + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 + gcc-native-mixed/12.3 @@ -718,20 +718,13 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 - evp-patch diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake new file mode 100644 index 000000000000..e79f70015471 --- /dev/null +++ b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake @@ -0,0 +1,12 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Remove this if you are using a resource manager (slurm etc) +set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") + +# EKAT MPI settings +set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index 079000059d19..e79f70015471 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) - # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 9536d415c0de..43e5ebfa083c 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,6 +82,10 @@ ["mpicxx","mpifort","mpicc"], "", ""), + "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index eaf65a69cda5..74579df552bd 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,14 +10,14 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" -#include "ElementOps.hpp" -#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" -#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" +// SCREAM includes +#include "share/util/scream_common_physics_functions.hpp" + // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -222,11 +222,7 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - - using EOS = Homme::EquationOfState; - using ElementOps = Homme::ElementOps; - using KV = Homme::KernelVariables; - + using PF = PhysicsFunctions; using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -263,7 +259,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states and EOS/EO classes + // Homme element states auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -272,11 +268,6 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - EOS eos; - eos.init(params.theta_hydrostatic_mode, hvcoord); - - ElementOps elem_ops; - elem_ops.init(hvcoord); const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); // Load data from IOP files, if necessary @@ -319,92 +310,66 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for both homme and scream - // related loops. We need separate policies since hommexx functions used here - // assume they are called inside nested loops for elements and Gaussian points, - // whereas EAMxx function we use expects a single level of parallelism - // for elements and Guassian points. - // TODO: scream::ColumnOps functions could take an arbitary loop boundary - // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that - // all 3 kernel launches here could be combined. - const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); - const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + // Team policy and workspace manager for eamxx + const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); - WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); + WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); view_Nd - rstar ("rstar", nelem, NGP, NGP, NLEV), - exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing rstar, exner, and temperature from Hommexx - auto compute_homme_states = [&] () { - Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + // Lambda for computing temperature + auto compute_temperature = [&] () { + Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = homme_wsm.get_workspace(team); - auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); - uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto pnh_i = ekat::subview(pnh, idx); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); - Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); - Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - - // Compute exner from EOS - if (theta_hydrostatic_mode) { - auto hydro_p_int = ws.take("hydro_p_int"); - Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); - elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); - eos.compute_exner(kv, pnh_scalar, exner_scalar); - ws.release(hydro_p_int); - } else { - eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); - } - - // Get the temperature from dynamics states - elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + // Compute temperature from virtual potential temperature + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { + auto T_val = vtheta_dp_i(k); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); }); // Release WS views - ws.release_macro_block(pnh_slot, NGP*NGP); + ws.release_many_contiguous<1>({&pmid}); }); }; - // Preprocess some homme states to get temperature and exner - compute_homme_states(); + // Preprocess some homme states to get temperature + compute_temperature(); Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -449,44 +414,57 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; + // Get temp views from workspace + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + + team.team_barrier(); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Recompute rstar with updated qv, and convert updated temperature back to potential temperature - elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); - }); + // Convert updated temperature back to psuedo density virtual potential temperature + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); + + // Release WS views + ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); if (iop_nudge_tq or iop_nudge_uv) { @@ -494,8 +472,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute rstar, exner and temperature from Hommexx - compute_homme_states(); + // Compute temperature + compute_temperature(); Kokkos::fence(); } @@ -571,25 +549,36 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_homme, + policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { + + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; + + // Get temp views from workspace + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + if (iop_nudge_tq or iop_nudge_uv) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high @@ -599,22 +588,26 @@ apply_iop_forcing(const Real dt) for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + and + pressure_from_iop >= iop_nudge_tq_high*100); } qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to potential temperature - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); } }); - }); + } + + // Release WS views + ws.release_many_contiguous<1>({&pmid}); }); } } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7ea2adfddb63..f8a34bd90d5b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,6 +317,8 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index d75af1cb2edb..ea0a18411c10 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=256), intent(inout) :: string + character(kind=c_char, len=512), intent(out) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,10 +69,17 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. - call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") - call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") - call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") - call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") +#ifdef SCREAM_DOUBLE_PRECISION + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR +#else + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR +#endif if (write_tables) then call p3_init_b() diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 9fb5726084f4..37cb251d7796 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 4cb3f50403ac..a105e244d76f 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,11 +1038,42 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { + std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - scorpio::define_var(filename, name, "unitless", vec_of_dims, - "real",fp_precision, m_add_time_dim); + if (mode==scorpio::FileMode::Append) { + // Similar to the regular fields above, check that the var is in the file, and has the right properties + EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), + "Error! Cannot append, due to variable missing from the file.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n"); + const auto& var = scorpio::get_var(filename,name); + EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, + "Error! Cannot append, due to variable dimensions mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var dims : " + ekat::join(vec_of_dims,",") + "\n" + " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + EKAT_REQUIRE_MSG (var.units==unitless, + "Error! Cannot append, due to variable units mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var units: " + unitless + "\n" + " - var units from file: " + var.units + "\n"); + EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, + "Error! Cannot append, due to time dependency mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" + " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); + } else { + // Note, unlike with regular output variables, for the average counting + // variables we don't need to add all of the extra metadata. So we simply + // define the variable. + scorpio::define_var(filename, name, unitless, vec_of_dims, + "real",fp_precision, m_add_time_dim); + } } } } // register_variables diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 2518907ec527..8d2f64994ddd 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -361,9 +361,9 @@ void finalize_subsystem () "Error! PIO subsystem was already finalized.\n"); for (auto& it : s.files) { - // EKAT_REQUIRE_MSG (it.second.num_customers==0, - // "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" - // " - filename: " + it.first + "\n"); + EKAT_REQUIRE_MSG (it.second.num_customers==0, + "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" + " - filename: " + it.first + "\n"); } s.files.clear(); diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 76e039e65ab5..32b120c57002 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -270,9 +270,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - m_params.get("target_latitude"); + std::max(m_params.get("target_latitude"),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - m_params.get("target_longitude"); + std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -543,6 +543,14 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); + // Read in IOP lev data + auto data = iop_file_pressure.get_view().data(); + scorpio::read_var(iop_file,"lev",data); + + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilev(); diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt index 50f2cfa68a3b..0a3a0b877c5e 100644 --- a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -40,7 +40,6 @@ set (TEST_INPUT_FILES scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc scream/mam4xx/physprops/poly_rrtmg_c20240206.nc - # scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) From 1d796b40a8362b503206274cfba423dd5c4577ee Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 15 Aug 2024 10:00:57 -0700 Subject: [PATCH 335/477] Changes emission files that use conservative mapping --- .../cime_config/namelist_defaults_scream.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 12095edff394..87791d0b7d00 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -271,14 +271,14 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc From 02b3d75d1765c30e6cd927f48776e111dd7ba1bb Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 16 Aug 2024 10:59:56 -0400 Subject: [PATCH 336/477] fix shoc warning unused vars --- .../src/physics/shoc/eamxx_shoc_process_interface.hpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 1695e60dfbc1..2a71870be9de 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -88,13 +88,10 @@ class SHOCMacrophysics : public scream::AtmosphereProcess cldfrac_liq_prev(i,k)=cldfrac_liq(i,k); - const auto range = ekat::range(k*Spack::n); - const Smask in_nlev_range = (range < nlev); - - // Inverse of Exner. Assert that exner != 0 when in range before computing. + // Inverse of Exner. In non-rel builds, assert that exner != 0 when in range before computing. const Spack exner = PF::exner_function(p_mid(i,k)); const Smask nonzero = (exner != 0); - EKAT_KERNEL_ASSERT((nonzero || !in_nlev_range).all()); + EKAT_KERNEL_ASSERT((nonzero || !(ekat::range(k*Spack::n) < nlev);).all()); inv_exner(i,k).set(nonzero, 1/exner); tke(i,k) = ekat::max(mintke, tke(i,k)); From 62cfa47f7f2cec73b26e80785787717d75c8000a Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 16 Aug 2024 11:29:57 -0400 Subject: [PATCH 337/477] fix typo in assert --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 2a71870be9de..469c31de7af5 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -91,7 +91,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Inverse of Exner. In non-rel builds, assert that exner != 0 when in range before computing. const Spack exner = PF::exner_function(p_mid(i,k)); const Smask nonzero = (exner != 0); - EKAT_KERNEL_ASSERT((nonzero || !(ekat::range(k*Spack::n) < nlev);).all()); + EKAT_KERNEL_ASSERT((nonzero || !(ekat::range(k*Spack::n) < nlev)).all()); inv_exner(i,k).set(nonzero, 1/exner); tke(i,k) = ekat::max(mintke, tke(i,k)); From 36936e52dddcfeb99a3f5976d3f3399833051328 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 14 Aug 2024 18:15:02 -0500 Subject: [PATCH 338/477] move to nb --- .../src/python/libpyscream/CMakeLists.txt | 9 ++- .../src/python/libpyscream/pyatmproc.hpp | 29 ++++--- .../eamxx/src/python/libpyscream/pyfield.hpp | 35 ++++---- .../eamxx/src/python/libpyscream/pygrid.hpp | 11 ++- .../src/python/libpyscream/pyparamlist.hpp | 81 ++++++++++--------- .../src/python/libpyscream/pyscream_ext.cpp | 20 ++--- .../eamxx/src/python/libpyscream/pyutils.hpp | 6 +- 7 files changed, 106 insertions(+), 85 deletions(-) diff --git a/components/eamxx/src/python/libpyscream/CMakeLists.txt b/components/eamxx/src/python/libpyscream/CMakeLists.txt index 103c6a80b888..7569d1cf7e60 100644 --- a/components/eamxx/src/python/libpyscream/CMakeLists.txt +++ b/components/eamxx/src/python/libpyscream/CMakeLists.txt @@ -1,7 +1,12 @@ -find_package(pybind11 REQUIRED) +# Detect the installed nanobind package and import it into CMake +find_package(Python COMPONENTS Interpreter Development REQUIRED) +execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -m nanobind --cmake_dir + OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT) +find_package(nanobind REQUIRED) find_package(mpi4py REQUIRED) -pybind11_add_module(pyscream_ext pyscream_ext.cpp) +nanobind_add_module(pyscream_ext pyscream_ext.cpp) target_link_libraries(pyscream_ext PUBLIC mpi4py scream_share diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 108edc2551b3..219837c5f736 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -12,8 +12,11 @@ #include -#include -#include +#include +#include +#include + +namespace nb = nanobind; namespace scream { @@ -26,7 +29,7 @@ struct PyAtmProc { std::shared_ptr output_mgr; - PyAtmProc (const pybind11::dict& d, const std::string& name) + PyAtmProc (const nb::dict& d, const std::string& name) { PyParamList params(d,name); @@ -102,7 +105,7 @@ struct PyAtmProc { ap->initialize(t0,RunType::Initial); } - pybind11::list read_ic (const std::string& ic_filename) { + std::vector read_ic (const std::string& ic_filename) { // Get input fields, and read them from file (if present). // If field is not in the IC, user is responsible for setting // it to an initial value @@ -128,10 +131,10 @@ struct PyAtmProc { } scorpio::release_file(ic_filename); - return pybind11::cast(missing); + return missing; } - pybind11::list list_fields(std::string ftype) { + std::vector list_fields(std::string ftype) { std::vector fields_list; for (const auto& field_pair : fields) { const auto& field_identifier = field_pair.second.f.get_header().get_identifier(); @@ -144,18 +147,18 @@ struct PyAtmProc { fields_list.push_back(field_pair.first); } } - return pybind11::cast(fields_list); + return fields_list; } - pybind11::list list_all_fields() { + std::vector list_all_fields() { return list_fields("all"); } - pybind11::list list_required_fields() { + std::vector list_required_fields() { return list_fields("required"); } - pybind11::list list_computed_fields() { + std::vector list_computed_fields() { return list_fields("computed"); } @@ -194,10 +197,10 @@ struct PyAtmProc { }; // Register type in the py module -inline void pybind_pyatmproc(pybind11::module& m) +inline void nb_pyatmproc(nb::module_& m) { - pybind11::class_(m,"AtmProc") - .def(pybind11::init()) + nb::class_(m,"AtmProc") + .def(nb::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) .def("get_params",&PyAtmProc::get_params) diff --git a/components/eamxx/src/python/libpyscream/pyfield.hpp b/components/eamxx/src/python/libpyscream/pyfield.hpp index eeb4125cecc2..96dae4b3e454 100644 --- a/components/eamxx/src/python/libpyscream/pyfield.hpp +++ b/components/eamxx/src/python/libpyscream/pyfield.hpp @@ -4,9 +4,11 @@ #include "share/field/field.hpp" #include "share/field/field_utils.hpp" -#include -#include -#include +#include +#include +#include + +namespace nb = nanobind; namespace scream { @@ -24,7 +26,8 @@ struct PyField { f.allocate_view(); } - pybind11::array get () const { + template + nb::ndarray get () const { const auto& fh = f.get_header(); const auto& fid = fh.get_identifier(); @@ -39,10 +42,14 @@ struct PyField { // NOTE: since the field may be padded, the strides do not necessarily // match the dims. Also, the strides must be grabbed from the // actual view, since the layout doesn't know them. - pybind11::array::ShapeContainer shape (fid.get_layout().dims()); + int shape_t = f.rank(); + size_t shape[shape_t] = {0}; + for (int i=0; i strides; - pybind11::dtype dt; + nb::dlpack::dtype dt; switch (fid.data_type()) { case DataType::IntType: dt = get_dt_and_set_strides(strides); @@ -59,8 +66,8 @@ struct PyField { // NOTE: you MUST set the parent handle, or else you won't have view semantic auto data = f.get_internal_view_data_unsafe(); - auto this_obj = pybind11::cast(this); - return pybind11::array(dt,shape,strides,data,pybind11::handle(this_obj)); + auto this_obj = nb::cast(this); + return nb::ndarray(data, shape_t, shape, nb::handle(this_obj), strides.data(), dt); } void sync_to_host () { @@ -75,7 +82,7 @@ struct PyField { private: template - pybind11::dtype get_dt_and_set_strides (std::vector& strides) const + nb::dlpack::dtype get_dt_and_set_strides (std::vector& strides) const { strides.resize(f.rank()); switch (f.rank()) { @@ -126,15 +133,15 @@ struct PyField { " - field rnak: " + std::to_string(f.rank()) + "\n"); } - return pybind11::dtype::of(); + return nb::dtype(); } }; -inline void pybind_pyfield (pybind11::module& m) { +inline void nb_pyfield (nb::module_& m) { // Field class - pybind11::class_(m,"Field") - .def(pybind11::init<>()) - .def("get",&PyField::get) + nb::class_(m,"Field") + .def(nb::init<>()) + .def("get",&PyField::get) .def("sync_to_host",&PyField::sync_to_host) .def("sync_to_dev",&PyField::sync_to_dev) .def("print",&PyField::print); diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp index 1c0d56eb08b3..83cb78d6729e 100644 --- a/components/eamxx/src/python/libpyscream/pygrid.hpp +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -5,10 +5,13 @@ #include "pyscream_ext.hpp" -#include +#include +#include #include +namespace nb = nanobind; + namespace scream { inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) @@ -38,9 +41,9 @@ inline void create_grids_manager (int ncols, int nlevs) create_grids_manager(ncols,nlevs,""); } -inline void pybind_pygrid (pybind11::module& m) { - m.def("create_grids_manager",pybind11::overload_cast(&create_grids_manager)); - m.def("create_grids_manager",pybind11::overload_cast(&create_grids_manager)); +inline void nb_pygrid (nb::module_& m) { + m.def("create_grids_manager",nb::overload_cast(&create_grids_manager)); + m.def("create_grids_manager",nb::overload_cast(&create_grids_manager)); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pyparamlist.hpp b/components/eamxx/src/python/libpyscream/pyparamlist.hpp index a631b6622e90..8da57a24bd35 100644 --- a/components/eamxx/src/python/libpyscream/pyparamlist.hpp +++ b/components/eamxx/src/python/libpyscream/pyparamlist.hpp @@ -3,12 +3,15 @@ #include -#include -#include -#include +#include +#include +#include +#include #include +namespace nb = nanobind; + namespace scream { struct PyParamList { @@ -19,11 +22,11 @@ struct PyParamList { : pl_ref(src) {} - PyParamList(const pybind11::dict& d) + PyParamList(const nb::dict& d) : PyParamList(d,"") {} - PyParamList(const pybind11::dict& d, const std::string& name) + PyParamList(const nb::dict& d, const std::string& name) : pl(name) , pl_ref(pl) { @@ -69,26 +72,26 @@ struct PyParamList { private: - void parse_dict(const pybind11::dict& d, ekat::ParameterList& p) { + void parse_dict(const nb::dict& d, ekat::ParameterList& p) { for (auto item : d) { - const std::string key = pybind11::str(item.first); - if (pybind11::isinstance(item.second)) { - auto pystr = pybind11::str(item.second); - p.set(key,pystr.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pyint = pybind11::cast(item.second); - p.set(key,pyint.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pyint = pybind11::cast(item.second); - p.set(key,pyint.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pydouble = pybind11::cast(item.second); - p.set(key,pydouble.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pylist = pybind11::cast(item.second); + auto key = nb::cast(item.first); + if (nb::isinstance(item.second)) { + auto pystr = nb::str(item.second); + p.set(key,nb::cast(pystr)); + } else if (nb::isinstance(item.second)) { + auto pyint = nb::cast(item.second); + p.set(key,nb::cast(pyint)); + } else if (nb::isinstance(item.second)) { + auto pyint = nb::cast(item.second); + p.set(key,nb::cast(pyint)); + } else if (nb::isinstance(item.second)) { + auto pydouble = nb::cast(item.second); + p.set(key,nb::cast(pydouble)); + } else if (nb::isinstance(item.second)) { + auto pylist = nb::cast(item.second); parse_list(pylist,p,key); - } else if (pybind11::isinstance(item.second)) { - auto pydict = pybind11::cast(item.second); + } else if (nb::isinstance(item.second)) { + auto pydict = nb::cast(item.second); parse_dict(pydict,p.sublist(key)); } else { EKAT_ERROR_MSG ("Unsupported/unrecognized dict entry type.\n"); @@ -96,43 +99,43 @@ struct PyParamList { } } - void parse_list (const pybind11::list& l, ekat::ParameterList&p, const std::string& key) { - EKAT_REQUIRE_MSG (pybind11::len(l)>0, + void parse_list (const nb::list& l, ekat::ParameterList&p, const std::string& key) { + EKAT_REQUIRE_MSG (nb::len(l)>0, "Error! Cannot deduce type for dictionary list entry '" + key + "'\n"); auto first = l[0]; - bool are_ints = pybind11::isinstance(first); - bool are_floats = pybind11::isinstance(first); - bool are_strings = pybind11::isinstance(first); + bool are_ints = nb::isinstance(first); + bool are_floats = nb::isinstance(first); + bool are_strings = nb::isinstance(first); if (are_ints) { - parse_list_impl(l,p,key); + parse_list_impl(l,p,key); } else if (are_floats) { - parse_list_impl(l,p,key); + parse_list_impl(l,p,key); } else if (are_strings) { - parse_list_impl(l,p,key); + parse_list_impl(l,p,key); } else { EKAT_ERROR_MSG ("Unrecognized/unsupported list entry type.\n"); } } template - void parse_list_impl(const pybind11::list& l, ekat::ParameterList& p, const std::string& key) { + void parse_list_impl(const nb::list& l, ekat::ParameterList& p, const std::string& key) { std::vector vals; for (auto item : l) { - EKAT_REQUIRE_MSG (pybind11::isinstance(item), + EKAT_REQUIRE_MSG (nb::isinstance(item), "Error! Inconsistent types in list entries.\n"); - auto item_py = pybind11::cast(item); - vals.push_back(item_py.template cast()); + auto item_py = nb::cast(item); + vals.push_back(nb::cast(item_py)); } p.set(key,vals); } }; -inline void pybind_pyparamlist (pybind11::module& m) +inline void nb_pyparamlist (nb::module_& m) { // Param list - pybind11::class_(m,"ParameterList") - .def(pybind11::init()) - .def(pybind11::init()) + nb::class_(m,"ParameterList") + .def(nb::init()) + .def(nb::init()) .def("sublist",&PyParamList::sublist) .def("print",&PyParamList::print) .def("set",&PyParamList::set) diff --git a/components/eamxx/src/python/libpyscream/pyscream_ext.cpp b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp index e0ad9ceec260..892d40e8ca3c 100644 --- a/components/eamxx/src/python/libpyscream/pyscream_ext.cpp +++ b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp @@ -7,11 +7,11 @@ #include -#include +#include #include -namespace py = pybind11; +namespace nb = nanobind; namespace scream { @@ -33,7 +33,7 @@ void initialize (MPI_Comm mpi_comm) { void initialize () { initialize(MPI_COMM_WORLD); } -void initialize (pybind11::object py_comm) { +void initialize (nb::object py_comm) { initialize(get_c_comm(py_comm)); } void finalize () { @@ -45,20 +45,20 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyscream_ext,m) { +NB_MODULE (pyscream_ext,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; // Scream Session - m.def("init",py::overload_cast<>(&initialize)); - m.def("init",py::overload_cast(&initialize)); + m.def("init",nb::overload_cast<>(&initialize)); + m.def("init",nb::overload_cast(&initialize)); m.def("finalize",&finalize); // Call all other headers' registration routines - pybind_pyparamlist(m); - pybind_pyfield(m); - pybind_pygrid(m); - pybind_pyatmproc(m); + nb_pyparamlist(m); + nb_pyfield(m); + nb_pygrid(m); + nb_pyatmproc(m); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pyutils.hpp b/components/eamxx/src/python/libpyscream/pyutils.hpp index 6467df57eda1..b02ec9a36a12 100644 --- a/components/eamxx/src/python/libpyscream/pyutils.hpp +++ b/components/eamxx/src/python/libpyscream/pyutils.hpp @@ -1,15 +1,15 @@ #ifndef PYUTILS_HPP #define PYUTILS_HPP -#include +#include #include #include #include -MPI_Comm get_c_comm (pybind11::object py_comm) { +MPI_Comm get_c_comm (nb::object py_comm) { if (import_mpi4py() < 0) { - throw pybind11::error_already_set(); + throw nb::python_error(); } auto py_src = py_comm.ptr(); if (not PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { From d8ae8f74bdbaa1168db0321c428a62f323e98808 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 16 Aug 2024 12:55:02 -0500 Subject: [PATCH 339/477] use EAMXX_ENABLE_PYSCREAM --- components/eamxx/CMakeLists.txt | 4 ++-- components/eamxx/src/CMakeLists.txt | 2 +- components/eamxx/tests/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 580ce26e7f50..398efa3bc2df 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -272,8 +272,8 @@ set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) -option (EAMXX_ENABLE_PYBIND "Whether to enable python interface to eamxx, via pybind11" OFF) -if (EAMXX_ENABLE_PYBIND) +option (EAMXX_ENABLE_PYSCREAM "Whether to enable python interface to eamxx" OFF) +if (EAMXX_ENABLE_PYSCREAM) # Pybind11 requires shared libraries set (BUILD_SHARED_LIBS ON) endif() diff --git a/components/eamxx/src/CMakeLists.txt b/components/eamxx/src/CMakeLists.txt index 7672d79f13d6..be88a2f51ae0 100644 --- a/components/eamxx/src/CMakeLists.txt +++ b/components/eamxx/src/CMakeLists.txt @@ -8,6 +8,6 @@ if (PROJECT_NAME STREQUAL "E3SM") add_subdirectory(mct_coupling) endif() -if (EAMXX_ENABLE_PYBIND) +if (EAMXX_ENABLE_PYSCREAM) add_subdirectory(python) endif() diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 7602f654356d..ad80045a0c13 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -75,7 +75,7 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) add_subdirectory(multi-process) endif() - if (EAMXX_ENABLE_PYBIND) + if (EAMXX_ENABLE_PYSCREAM) add_subdirectory(python) endif() endif() From 731618181e6c91c92335eb1b535f706ed654657a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 11:05:22 -0700 Subject: [PATCH 340/477] Fixes multi process test a typo in aci interface --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 8 ++++---- .../output.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7800a43c252d..ef2c0f2b2882 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -253,7 +253,7 @@ void MAMAci::set_grids( frz_unit, grid_name); // heterogeneous freezing by deposition nucleation [cm^-3 s^-1] - add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, + add_field("hetfrz_deposition_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); } // function set_grids ends @@ -390,8 +390,8 @@ void MAMAci::initialize_impl(const RunType run_type) { get_field_out("hetfrz_immersion_nucleation_tend").get_view(); hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = - get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + hetfrz_deposition_nucleation_tend_ = + get_field_out("hetfrz_deposition_nucleation_tend").get_view(); //--------------------------------------------------------------------------------- // Allocate memory for the class members @@ -642,7 +642,7 @@ void MAMAci::run_impl(const double dt) { team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, // ## output to be used by the other processes ## hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, - hetfrz_depostion_nucleation_tend_, + hetfrz_deposition_nucleation_tend_, // work arrays diagnostic_scratch_); diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml index 15e60e91517c..c9cad5e2761a 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_mam4_drydep_output +filename_prefix: mam4_srf_online_emiss_mam4_constituent_fluxes_output Averaging Type: Instant Field Names: - bc_a1 From c0a608e419600b9260f62f75030c03a9149d459e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 15:32:22 -0700 Subject: [PATCH 341/477] Adds ne30pg2 emissions files and mapping files for fine resolutions --- .../cime_config/namelist_defaults_scream.xml | 18 ++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 87791d0b7d00..a785fb322912 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -279,6 +279,24 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + + + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120np4_mono_20220502.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index fc930ea11d7b..63f87dd9f246 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -123,7 +123,7 @@ class MAMAci final : public scream::AtmosphereProcess { // added correctly to the cloud-micorphysics scheme. view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; - view_2d hetfrz_depostion_nucleation_tend_; + view_2d hetfrz_deposition_nucleation_tend_; view_2d diagnostic_scratch_[hetro_scratch_]; From f35d4d75f6bc958667a524fda3587cb08462adbe Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 16 Aug 2024 19:39:58 -0400 Subject: [PATCH 342/477] disable gh testing for now --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 2 +- .github/workflows/e3sm-gh-ci-w-cime-tests.yml | 1 + .github/workflows/eamxx_default_files.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 366fef018f20..4cb89ed3cb9d 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -21,7 +21,7 @@ on: jobs: ci: - if: ${{ github.event.repository.name == 'e3sm' }} + if: false runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index 595518c326d7..2553c63cdb5f 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -14,6 +14,7 @@ on: jobs: ci-w: + if: false runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index e0a8e19f9c01..7f8f2a177515 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -11,6 +11,7 @@ on: jobs: scream-defaults: + if: false runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} From 4fa577ebced271a90ee7c29cb308eada86426d6f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 19:12:45 -0700 Subject: [PATCH 343/477] Removes debug print statements --- ...eamxx_mam_constituent_fluxes_interface.cpp | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index c74dbd7e4fa0..c3a33687dad6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -1,4 +1,3 @@ -#include #include #include namespace scream { @@ -282,30 +281,10 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - auto start = std::chrono::steady_clock::now(); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, constituent_fluxes_, // output wet_aero_); - auto stop = std::chrono::steady_clock::now(); - auto duration = (stop - start); - - // To get the value of duration use the count() - // member function on the duration object - printf("TIME:%e\n", - std::chrono::duration(duration).count()); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - } // run_impl ends // ============================================================================= From 796403f204752d74a724c6c8ffd2003d66ac9b06 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Sat, 17 Aug 2024 17:53:48 -0400 Subject: [PATCH 344/477] fix errors and warning in rrtmgp-k --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 22 +++++++++++++++---- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 6 ++--- .../rrtmgp/scream_rrtmgp_interface.hpp | 4 ++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 62de7f8416cd..e89e02fe301b 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -1171,10 +1171,24 @@ void RRTMGPRadiation::run_impl (const double dt) { aero_tau_lw_k(i,k,b) = d_aero_tau_lw(icol,b,k); }); } else { - Kokkos::deep_copy(aero_tau_sw_k, 0); - Kokkos::deep_copy(aero_ssa_sw_k, 0); - Kokkos::deep_copy(aero_g_sw_k , 0); - Kokkos::deep_copy(aero_tau_lw_k, 0); + // cuda complains (in warning only) about these being not allowed... + // Kokkos::deep_copy(aero_tau_sw_k, 0); + // Kokkos::deep_copy(aero_ssa_sw_k, 0); + // Kokkos::deep_copy(aero_g_sw_k , 0); + // Kokkos::deep_copy(aero_tau_lw_k, 0); + // So, do the manual labor instead: + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_sw_k(i,k,b) = 0.0; + aero_ssa_sw_k(i,k,b) = 0.0; + aero_g_sw_k (i,k,b) = 0.0; + }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlwbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_lw_k(i,k,b) = 0.0; + }); } #endif }); diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 8c236e64e811..aa6326c81811 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -112,7 +112,7 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template ::type* = nullptr> +template ::type* dummy = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; @@ -137,7 +137,7 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val return pass; } -template ::type* = nullptr> +template ::type* dummy = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; @@ -166,7 +166,7 @@ bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_val return pass; } -template ::type* = nullptr> +template ::type* dummy = nullptr> bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 0bceb1282e5f..216722b3766b 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -1190,14 +1190,14 @@ static void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -template::type* = nullptr> +template::type* dummy = nullptr> static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { Kokkos::parallel_for(arr_out.size(), KOKKOS_LAMBDA(int i) { arr_out(i) = std::min(std::max(arr_in(i), lower), upper); }); } -template::type* = nullptr> +template::type* dummy = nullptr> static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { Kokkos::parallel_for(MDRP::template get<2>({arr_out.extent(0), arr_out.extent(1)}), KOKKOS_LAMBDA(int i, int j) { arr_out(i, j) = std::min(std::max(arr_in(i, j), lower), upper); From 87ea43c4f5cefdf12bd1ee869c3508230db1f8b1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 17 Aug 2024 15:20:01 -0700 Subject: [PATCH 345/477] Edits and adds some comments to provide clarity and TODOs --- .../mam/eamxx_mam_constituent_fluxes_functions.hpp | 3 +++ .../mam/eamxx_mam_constituent_fluxes_interface.cpp | 8 ++++---- .../mam/eamxx_mam_constituent_fluxes_interface.hpp | 2 +- ...mxx_mam_srf_and_online_emissions_process_interface.cpp | 4 +++- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 2 ++ 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index e7ffb6376c35..12c9c172f17d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -33,6 +33,9 @@ void update_gas_aerosols_using_constituents( // Create a policy to loop over columns annd number of constituents // to update + // FIXME: TODO:We don't need a team for "nconstituents", so we can make the + // kookos_for + // simple by using just ncols const auto policy = ekat::ExeSpaceUtils:: get_default_team_policy(ncol, nconstituents); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index c3a33687dad6..96d26ca22f9e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -267,13 +267,13 @@ void MAMConstituentFluxes::run_impl(const double dt) { // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". + // mixing ratios), we are *not* doing any wet to dry conversions in the + // "preprocess" for this process. We are simply updating the MAM4xx + // tracers using the "constituent fluxes". // We are converting wet atm to dry atm. Since we do not use or update // any of the water constituents (qc, qv, qi etc.), we should be okay // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. + // are built following HAERO data structures. // ------------------------------------------------------------------- // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 7e77fccf2afc..c68065f21788 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -27,7 +27,7 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { mam_coupling::DryAtmosphere dry_atm_; // aerosol state variables - mam_coupling::AerosolState wet_aero_; //, dry_aero_; + mam_coupling::AerosolState wet_aero_; // buffer for sotring temporary variables mam_coupling::Buffer buffer_; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f2a68ee6b5a7..850a82d0896d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -42,6 +42,8 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file auto srf_map_file = m_params.get("srf_remap_file", ""); + // FIXME: We can extract the following info about each species + // in a separate hpp file //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- @@ -221,7 +223,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - // Zero output + // Zero-out output Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 2d29075e41a4..b8ac7de768db 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -28,6 +28,8 @@ srfEmissFunctions::create_horiz_remapper( const int ncols_model = model_grid->get_num_global_dofs(); std::shared_ptr remapper; + // if the file's grid is same as model's native grid, we identity remapper + // (i.e., no interpolation) if(ncols_data == ncols_model) { remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); From b3ffc69e30889dac9a9344617c9b9f6072ddca00 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 19 Aug 2024 10:49:08 -0600 Subject: [PATCH 346/477] Limit threads available to compute-sanitizer racecheck --- components/eamxx/scripts/test_all_scream.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index 2d1ee9378ff8..f5350954b1c3 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -2,7 +2,7 @@ SharedArea, safe_copy from git_utils import get_current_head, get_current_commit -from test_factory import create_tests, COV +from test_factory import create_tests, COV, CSR from machines_specs import get_mach_compilation_resources, get_mach_testing_resources, \ get_mach_baseline_root_dir, setup_mach_env, is_cuda_machine, \ @@ -386,6 +386,15 @@ def generate_cmake_config(self, test, for_ctest=False): if "SCREAM_DYNAMICS_DYCORE" not in custom_opts_keys: result += " -DSCREAM_DYNAMICS_DYCORE=HOMME" + # For the compute-sanitizer tool 'racecheck', if no option --racecheck-num-workers + # is provided, it will attempt to use all threads available on node. This can cause + # issues if other test cases are being run in parallel. If the option was not specified, + # limit the number of threads availible to racecheck to the number of compile resources. + if self._parallel and isinstance(test, CSR) and not '--racecheck-num-workers' in result: + new_option = ' --racecheck-num-workers=' + str(test.compile_res_count) + index = result.index('--tool=racecheck') + len('--tool=racecheck') + result = result[:index] + new_option + result[index:] + return result ############################################################################### From 4375b3144492349002c5294e0b863a4732144b20 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Mon, 19 Aug 2024 10:50:16 -0600 Subject: [PATCH 347/477] Add single quotes to CS options If we ever need to add more options, not having these quotes might lead to hard to find errors --- components/eamxx/scripts/test_factory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py index 0b3f0c738c88..f85461ad23b4 100644 --- a/components/eamxx/scripts/test_factory.py +++ b/components/eamxx/scripts/test_factory.py @@ -176,7 +176,7 @@ def __init__(self, _): "debug with compute sanitizer memcheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=memcheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -210,7 +210,7 @@ def __init__(self, _): "debug with compute sanitizer initcheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=initcheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=initcheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -227,7 +227,7 @@ def __init__(self, _): "debug with compute sanitizer synccheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=synccheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=synccheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" From 8fb6e06d920aacefbfbaed525c3f0bb737523430 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 15 Aug 2024 15:53:31 -0600 Subject: [PATCH 348/477] Use C++ in shoc property tests --- .../shoc/tests/shoc_assumed_pdf_tests.cpp | 42 ++++++++++++++----- .../shoc/tests/shoc_brunt_length_tests.cpp | 7 +++- .../shoc/tests/shoc_check_length_tests.cpp | 7 +++- .../shoc/tests/shoc_check_tke_tests.cpp | 7 +++- .../shoc/tests/shoc_clip_third_moms_tests.cpp | 7 +++- .../tests/shoc_compute_diag_third_tests.cpp | 11 ++++- .../shoc_compute_shoc_temperature_tests.cpp | 18 +++++--- .../tests/shoc_compute_shoc_vapor_tests.cpp | 6 ++- .../shoc/tests/shoc_diag_obklen_tests.cpp | 18 ++++++-- .../tests/shoc_diag_second_mom_srf_test.cpp | 4 +- .../shoc_diag_second_mom_ubycond_test.cpp | 5 ++- ...shoc_diag_second_moments_lbycond_tests.cpp | 7 +++- .../tests/shoc_diag_second_moments_tests.cpp | 9 +++- .../shoc_diag_second_shoc_moments_tests.cpp | 9 +++- .../shoc/tests/shoc_diag_third_tests.cpp | 20 +++++++-- .../tests/shoc_eddy_diffusivities_tests.cpp | 21 +++++++--- .../shoc/tests/shoc_energy_fixer_tests.cpp | 22 ++++++++-- .../shoc/tests/shoc_energy_integral_tests.cpp | 9 +++- .../tests/shoc_energy_update_dse_tests.cpp | 8 +++- .../physics/shoc/tests/shoc_grid_tests.cpp | 6 ++- .../shoc/tests/shoc_impli_comp_tmpi_tests.cpp | 6 ++- .../tests/shoc_impli_dp_inverse_tests.cpp | 6 ++- .../shoc/tests/shoc_l_inf_length_tests.cpp | 7 +++- .../physics/shoc/tests/shoc_length_tests.cpp | 18 ++++++-- .../shoc/tests/shoc_linear_interp_tests.cpp | 16 +++++-- .../physics/shoc/tests/shoc_main_tests.cpp | 15 ++++++- .../shoc/tests/shoc_mix_length_tests.cpp | 10 ++++- .../tests/shoc_pblintd_check_pblh_tests.cpp | 6 ++- .../tests/shoc_pblintd_cldcheck_tests.cpp | 5 ++- .../shoc/tests/shoc_pblintd_height_tests.cpp | 21 +++++++--- .../shoc/tests/shoc_pblintd_init_pot_test.cpp | 8 ++-- .../tests/shoc_pblintd_surf_temp_tests.cpp | 7 +++- .../physics/shoc/tests/shoc_pblintd_tests.cpp | 8 +++- .../shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp | 12 ++++-- .../shoc/tests/shoc_tke_column_stab_tests.cpp | 12 ++++-- .../tests/shoc_tke_isotropic_ts_tests.cpp | 12 ++++-- .../shoc/tests/shoc_tke_shr_prod_tests.cpp | 12 ++++-- .../src/physics/shoc/tests/shoc_tke_tests.cpp | 16 +++++-- ...shoc_update_prognostics_implicit_tests.cpp | 14 +++++-- .../shoc/tests/shoc_varorcovar_tests.cpp | 30 ++++++++++--- .../shoc/tests/shoc_vertflux_tests.cpp | 7 +++- 41 files changed, 362 insertions(+), 129 deletions(-) diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index 3f1ab03f1a03..3672101c7dd5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -102,8 +102,6 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } // Check that the inputs make sense - - // Load input data for(Int s = 0; s < shcol; ++s) { for(Int n = 0; n < nlev; ++n) { const auto offset = n + s * nlev; @@ -142,8 +140,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { // For this test we want exactly two columns REQUIRE(SDS.shcol == 2); - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Verify the result // Make sure cloud fraction is either 1 or 0 and all @@ -182,8 +186,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Verify the result // Make sure cloud fraction is either 1 or 0 and all @@ -240,8 +250,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Check the result @@ -324,8 +340,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp index 4c9ff408e675..3018d0e5aa6a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp @@ -90,8 +90,11 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { } } - // Call the fortran implementation - compute_brunt_shoc_length(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + compute_brunt_shoc_length_f(SDS.nlev,SDS.nlevi,SDS.shcol,SDS.dz_zt,SDS.thv,SDS.thv_zi,SDS.brunt); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp index 1069b51d408b..bd023e2ed794 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp @@ -75,8 +75,11 @@ struct UnitWrap::UnitTest::TestCheckShocLength { REQUIRE(SDS.host_dy[s] > 0); } - // Call the fortran implementation - check_length_scale_shoc_length(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + check_length_scale_shoc_length_f(SDS.nlev,SDS.shcol,SDS.host_dx,SDS.host_dy,SDS.shoc_mix); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp index ca22ffe3b2ef..75581e2283ce 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp @@ -52,8 +52,11 @@ struct UnitWrap::UnitTest::TestShocCheckTke { // Check some input REQUIRE((SDS.shcol > 0 && SDS.nlev > 0)); - // call the fortran implementation - check_tke(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + check_tke_f(SDS.nlev, SDS.shcol, SDS.tke); + SDS.transpose(); // Check the result against the input values for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp index ee8eb4df9523..28bef1aae259 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp @@ -82,8 +82,11 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { REQUIRE(w3_large == true); } - // Call the fortran implementation - clipping_diag_third_shoc_moments(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + clipping_diag_third_shoc_moments_f(SDS.nlevi,SDS.shcol,SDS.w_sec_zi,SDS.w3); + SDS.transpose(); // Check the result // For large values of w3, verify that the result has been reduced diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp index 0e58b658d475..d63bf34bb50e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp @@ -145,8 +145,15 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { } } - // Call the fortran implementation - compute_diag_third_shoc_moment(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_diag_third_shoc_moment_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.tke,SDS.dz_zt, + SDS.dz_zi,SDS.isotropy_zi, + SDS.brunt_zi,SDS.w_sec_zi,SDS.thetal_zi, + SDS.w3); + SDS.transpose(); // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp index ae359978cf32..27452cdcf703 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp @@ -67,8 +67,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature is equal to thetal for(Int s = 0; s < shcol; ++s) { @@ -113,8 +115,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature is greather than thetal for(Int s = 0; s < shcol; ++s) { @@ -172,8 +176,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature be less than thetal for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp index 004f072405ba..afd184e18231 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp @@ -62,8 +62,10 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { } } - // Call the fortran implementation - compute_shoc_vapor(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_vapor_f(SDS.shcol, SDS.nlev, SDS.qw, SDS.ql, SDS.qv); + SDS.transpose(); // go back to C layout // Verify the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp index 3662cc2741a0..930faf31d6b2 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp @@ -83,8 +83,13 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { REQUIRE( (SDS.qv_sfc[s] > 0 && SDS.qv_sfc[s] < 0.1) ); } - // Call the fortran implementation - shoc_diag_obklen(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, + SDS.obklen); + SDS.transpose(); // Check the result @@ -148,8 +153,13 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { SDS.wthl_sfc[s]+SDS.wqw_sfc[s]); } - // Call the fortran implementation - shoc_diag_obklen(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, + SDS.obklen); + SDS.transpose(); // Verify that DIMENSIONLESS obukhov length decreases as columns // increases due to the increasing surface fluxes diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp index 507103e59a1b..25f56139d776 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp @@ -51,8 +51,8 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { SDS.vw_sfc[s] = vw_sfc[s]; } - // Call the fortran implementation - diag_second_moments_srf(SDS); + // Call the C++ implementation + shoc_diag_second_moments_srf_f(SDS.shcol, SDS.wthl_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.ustar2, SDS.wstar); // Verify the output for (Int s = 0; s < shcol; ++s){ diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp index 68fccb11bfcb..d4dbd3e7b71f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp @@ -42,8 +42,9 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { REQUIRE(SDS.shcol == shcol); REQUIRE(shcol > 0); - // Call the fortran implementation - diag_second_moments_ubycond(SDS); + // Call the C++ implementation + shoc_diag_second_moments_ubycond_f(SDS.shcol, SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, + SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec); // Verify the result // all output should be zero. diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp index bd2a3ac60ee9..1f1f3df8e5cc 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp @@ -75,8 +75,11 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { REQUIRE(SDS.ustar2[s] >= 0); } - // Call the fortran implementation - diag_second_moments_lbycond(SDS); + // Call the C++ implementation + diag_second_moments_lbycond_f(SDS.shcol, SDS.wthl_sfc, SDS.wqw_sfc, SDS.uw_sfc, + SDS.vw_sfc, SDS.ustar2, SDS.wstar, SDS.wthl_sec, + SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, + SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec); // Verify output is as expected for (Int s = 0; s < shcol; ++s){ diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp index 1ff4df42d168..f83bfa313f9c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp @@ -162,8 +162,13 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { } } - // Call the fortran implementation - diag_second_moments(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + diag_second_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, + SDS.tke, SDS.isotropy, SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, SDS.uw_sec, + SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); + SDS.transpose(); // go back to C layout // Verify output makes sense for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp index 86084fef344a..0385335c808b 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp @@ -160,8 +160,13 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { } } - // Call the fortran implementation - diag_second_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + diag_second_shoc_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, SDS.tke, SDS.isotropy, + SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.uw_sfc, SDS.vw_sfc, SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, + SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); + SDS.transpose(); // go back to C layout // Verify output makes sense for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp index e6430fa00d7f..82b4d88c0f70 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp @@ -142,8 +142,14 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } } - // Call the fortran implementation - diag_third_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, + SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, + SDS.w3); + SDS.transpose(); // Check to make sure there is at least one // positive w3 value for convective boundary layer @@ -186,8 +192,14 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } } - // Call the fortran implementation - diag_third_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, + SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, + SDS.w3); + SDS.transpose(); // Verify that new result is greater or equal in magnitude // that the result from test one diff --git a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp index 4cb8d7a99c19..0a96e5adc629 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp @@ -97,8 +97,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the answers in the columns are different for(Int s = 0; s < shcol-1; ++s) { @@ -164,8 +167,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the answers in the columns are larger // when the length scale and shear term are larger @@ -234,8 +240,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the diffusivities are smaller // in the columns where isotropy and tke are smaller diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp index 8ebf2a69c88e..dfb37104f018 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp @@ -157,8 +157,15 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } } - // Call the fortran implementation - shoc_energy_fixer(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, + SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, + SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, + SDS.host_dse); + SDS.transpose(); // Check test // Verify that the dry static energy has not changed if surface @@ -231,8 +238,15 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } } - // Call the fortran implementation - shoc_energy_fixer(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, + SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, + SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, + SDS.host_dse); + SDS.transpose(); // Verify the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp index 705d5ad02f25..8142da9371da 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp @@ -107,8 +107,13 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { } } - // Call the fortran implementation - shoc_energy_integrals(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_integrals_f(SDS.shcol, SDS.nlev, SDS.host_dse, SDS.pdel, + SDS.rtm, SDS.rcm, SDS.u_wind, SDS.v_wind, + SDS.se_int, SDS.ke_int, SDS.wv_int, SDS.wl_int); + SDS.transpose(); // Check test for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp index 74d5094076b0..a4e01eadb852 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp @@ -108,8 +108,12 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { } } - // Call the fortran implementation - update_host_dse(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + update_host_dse_f(SDS.shcol,SDS.nlev,SDS.thlm,SDS.shoc_ql,SDS.inv_exner,SDS.zt_grid, + SDS.phis,SDS.host_dse); + SDS.transpose(); // Check test for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp index d14fdf199175..6a79eeca02ee 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp @@ -79,8 +79,10 @@ struct UnitWrap::UnitTest::TestShocGrid { } } - // Call the fortran implementation - shoc_grid(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_grid_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zt_grid, SDS.zi_grid, SDS.pdel, SDS.dz_zt, SDS.dz_zi, SDS.rho_zt); + SDS.transpose(); // go back to C layout // First check that dz is correct for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp index 6b641b57d61d..53280dab17a6 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp @@ -86,8 +86,10 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { } } - // Call the fortran implementation - compute_tmpi(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_tmpi_f(SDS.nlevi, SDS.shcol, SDS.dtime, SDS.rho_zi, SDS.dz_zi, SDS.tmpi); + SDS.transpose(); // go back to C layout // Verify result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp index e2fabf24cb36..aad9c0266869 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp @@ -73,8 +73,10 @@ struct UnitWrap::UnitTest::TestImpDpInverse { } } - // Call the fortran implementation - dp_inverse(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + dp_inverse_f(SDS.nlev, SDS.shcol, SDS.rho_zt, SDS.dz_zt, SDS.rdp_zt); + SDS.transpose(); // go back to C layout // Verify result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp index dee0e30beda4..84faaebe4b91 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp @@ -88,8 +88,11 @@ struct UnitWrap::UnitTest::TestLInfShocLength { } } - // Call the fortran implementation - compute_l_inf_shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_l_inf_shoc_length_f(SDS.nlev,SDS.shcol,SDS.zt_grid,SDS.dz_zt,SDS.tke,SDS.l_inf); + SDS.transpose(); // Check the results // Make sure result is bounded correctly diff --git a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp index 873feffeb455..0f890db9ce11 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp @@ -120,8 +120,13 @@ struct UnitWrap::UnitTest::TestShocLength { } } - // Call the Fortran implementation - shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, + SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, + SDS.thv,SDS.brunt,SDS.shoc_mix); + SDS.transpose(); // Verify output for(Int s = 0; s < shcol; ++s) { @@ -169,8 +174,13 @@ struct UnitWrap::UnitTest::TestShocLength { SDS.host_dy[s] = host_dy_small; } - // call fortran implentation - shoc_length(SDS); + // call C++ implentation + SDS.transpose(); + // expects data in fortran layout + shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, + SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, + SDS.thv,SDS.brunt,SDS.shoc_mix); + SDS.transpose(); // Verify output for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp index 2c55de1433df..0a9c666688b5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp @@ -99,8 +99,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - // Call the fortran implementation - linear_interp(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + linear_interp_f(SDS.x1, SDS.x2, SDS.y1, SDS.y2, SDS.km1, SDS.km2, SDS.ncol, SDS.minthresh); + SDS.transpose(); // go back to C layout // First check that all output temperatures are greater than zero @@ -191,7 +193,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - linear_interp(SDS2); + // Call the C++ implementation + SDS2.transpose(); // _f expects data in fortran layout + linear_interp_f(SDS2.x1, SDS2.x2, SDS2.y1, SDS2.y2, SDS2.km1, SDS2.km2, SDS2.ncol, SDS2.minthresh); + SDS2.transpose(); // go back to C layout // Check the result, make sure output is bounded correctly @@ -275,7 +280,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - linear_interp(d); + // Call the C++ implementation + d.transpose(); // _f expects data in fortran layout + linear_interp_f(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); + d.transpose(); // go back to C layout // The combination of single-precision and randomness generating points // close together can result in larger error margins. diff --git a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp index e699c3283284..5c3eb8f9860e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp @@ -258,8 +258,19 @@ struct UnitWrap::UnitTest::TestShocMain { } - // Call the fortran implementation - shoc_main(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + const int npbl = shoc_init_f(SDS.nlev, SDS.pref_mid, SDS.nbot_shoc, SDS.ntop_shoc); + + shoc_main_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, npbl, SDS.host_dx, SDS.host_dy, + SDS.thv, SDS.zt_grid, SDS.zi_grid, SDS.pres, SDS.presi, SDS.pdel, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.wtracer_sfc, SDS.num_qtracers, + SDS.w_field, SDS.inv_exner, SDS.phis, SDS.host_dse, SDS.tke, SDS.thetal, SDS.qw, + SDS.u_wind, SDS.v_wind, SDS.qtracers, SDS.wthv_sec, SDS.tkh, SDS.tk, SDS.shoc_ql, + SDS.shoc_cldfrac, SDS.pblh, SDS.shoc_mix, SDS.isotropy, SDS.w_sec, SDS.thl_sec, + SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.wtke_sec, SDS.uw_sec, + SDS.vw_sec, SDS.w3, SDS.wqls_sec, SDS.brunt, SDS.shoc_ql2); + SDS.transpose(); // go back to C layout // Make sure output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp index 40303c6d0487..4d6b5dfdce1c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp @@ -90,8 +90,14 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { } } - // Call the fortran implementation - compute_shoc_mix_shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_shoc_mix_shoc_length_f(SDS.nlev, SDS.shcol, + SDS.tke, SDS.brunt, + SDS.zt_grid, + SDS.l_inf, SDS.shoc_mix); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp index 7ee84767a938..2d5f01c8ff27 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp @@ -67,8 +67,10 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { } } - // Call the fortran implementation - pblintd_check_pblh(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_check_pblh_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.nlev, SDS.z, SDS.ustar, SDS.check, SDS.pblh); + SDS.transpose(); // go back to C layout // Check the result // Check that PBL height is greater than zero. This is an diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp index 41b61778630c..df65050acacc 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp @@ -79,8 +79,9 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { } } - // Call the fortran implementation - pblintd_cldcheck(SDS); + // Call the C++ implementation + SDS.transpose(); + shoc_pblintd_cldcheck_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zi, SDS.cldn, SDS.pblh); // Check the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp index ca30fc4db5b0..9c33c7de68be 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp @@ -85,8 +85,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { REQUIRE(SDS.ustar[s+1] > SDS.ustar[s]); } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { @@ -119,8 +122,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { SDS.thv[offset] = SDS.thv[offset-1] + 2; } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { @@ -158,8 +164,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check that PBLH is zero (not modified) everywhere for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp index 910f71836ea7..b01c39eed95a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp @@ -76,8 +76,8 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } } - // call the fortran implementation - pblintd_init_pot(SDS); + // call the C++ implementation + shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); // Check the result. // Verify that virtual potential temperature is idential @@ -125,8 +125,8 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } } - // Call the fortran implementation - pblintd_init_pot(SDS); + // Call the C++ implementation + shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); // Check test // Verify that column with condensate loading diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp index 55e11953db02..ccf3d110fc8d 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp @@ -83,8 +83,11 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { } } - // Call the fortran implementation - pblintd_surf_temp(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_surf_temp_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.z, SDS.ustar, SDS.obklen, + SDS.kbfs, SDS.thv, SDS.tlv, SDS.pblh, SDS.check, SDS.rino); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp index 7e27f8cce296..b5cd117228bf 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp @@ -121,8 +121,12 @@ struct UnitWrap::UnitTest::TestPblintd { } } - // Call the fortran implementation - pblintd(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.npbl, SDS.z, SDS.zi, + SDS.thl, SDS.ql, SDS.q, SDS.u, SDS.v, SDS.ustar, SDS.obklen, + SDS.kbfs, SDS.cldn, SDS.pblh); + SDS.transpose(); // go back to C layout // Make sure PBL height is reasonable // Should be larger than second lowest zi level and lower diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp index 1434cb27f4ec..ea2c95470d89 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp @@ -94,8 +94,10 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } } - // Call the fortran implementation - adv_sgs_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); + SDS.transpose(); // go back to C layout // Check to make sure that there has been // TKE growth @@ -159,8 +161,10 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } } - // Call the fortran implementation - adv_sgs_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); + SDS.transpose(); // go back to C layout // Check to make sure that the column with // the smallest length scale has larger diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp index 7cbb00255a76..eb68bbc11df1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp @@ -75,8 +75,10 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } } - // Call the fortran implementation - integ_column_stability(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); + SDS.transpose(); // go back to C layout // Check test // Verify that output is zero @@ -107,8 +109,10 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } } - // Call the fortran implementation - integ_column_stability(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); + SDS.transpose(); // go back to C layout // Check test // Verify that output is negative diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp index 0baeb12b029a..c90128857cbb 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp @@ -81,8 +81,10 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } } - // Call the fortran implementation - isotropic_ts(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -145,8 +147,10 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } } - // Call the fortran implementation - isotropic_ts(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp index 8b088852af9c..0672ce3b3ae6 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp @@ -93,8 +93,10 @@ struct UnitWrap::UnitTest::TestShocShearProd { } } - // Call the fortran implementation - compute_shr_prod(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); + SDS.transpose(); // go back to C layout // Check test for(Int s = 0; s < shcol; ++s) { @@ -142,8 +144,10 @@ struct UnitWrap::UnitTest::TestShocShearProd { } } - // Call the fortran implementation - compute_shr_prod(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); + SDS.transpose(); // go back to C layout // Check test // Verify that shear term is zero everywhere diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp index bc975c9b59c1..f09ecc79c7d3 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp @@ -153,8 +153,12 @@ struct UnitWrap::UnitTest::TestShocTke { } } - // Call the fortran implementation - shoc_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, + SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, + SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check test // Make sure that TKE has increased everwhere relative @@ -226,8 +230,12 @@ struct UnitWrap::UnitTest::TestShocTke { } } - // Call the fortran implementation - shoc_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, + SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, + SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp index 45268023faef..83cfb8d5707e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp @@ -258,10 +258,18 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { } } - // Call the fortran implementation - update_prognostics_implicit(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + update_prognostics_implicit_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.num_tracer, SDS.dtime, + SDS.dz_zt, SDS.dz_zi, SDS.rho_zt, SDS.zt_grid, SDS.zi_grid, + SDS.tk, SDS.tkh, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.wtracer_sfc, SDS.thetal, SDS.qw, SDS.tracer, SDS.tke, SDS.u_wind, SDS.v_wind); + SDS.transpose(); // go back to C layout + // Call linear interp to get rho value at surface for checking - linear_interp(SDSL); + SDSL.transpose(); // _f expects data in fortran layout + linear_interp_f(SDSL.x1, SDSL.x2, SDSL.y1, SDSL.y2, SDSL.km1, SDSL.km2, SDSL.ncol, SDSL.minthresh); + SDSL.transpose(); // go back to C layout // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp index 74990b2c43f2..4a743009f244 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp @@ -115,8 +115,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for variance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for variance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -172,8 +178,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for covariance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for covariance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -250,8 +262,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for variance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for variance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp index e8b1b5a8cdcd..7ac7c7c53160 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp @@ -95,8 +95,11 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { } } - // Call the fortran implementation - calc_shoc_vertflux(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + calc_shoc_vertflux_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.tkh_zi, SDS.dz_zi, SDS.invar, SDS.vertflux); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { From f8a137341ecf1e2f653389c1130ca216a896c81d Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 20 Aug 2024 10:10:44 -0600 Subject: [PATCH 349/477] Use eps to compare shoc_ql2==0 --- .../eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index 3672101c7dd5..d53bf4935dbd 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -160,9 +160,8 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] == 0); - REQUIRE(SDS.shoc_ql2[offset] == 0); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); REQUIRE(SDS.shoc_ql[offset] >= 0); - } } @@ -207,7 +206,7 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] != 0); REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); - REQUIRE(SDS.shoc_ql2[offset] == 0); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); REQUIRE(SDS.shoc_ql[offset] >= 0); REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); } From 2aaccf32f6909456f5ce4fdc0e3ebcd89f81b3f7 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 20 Aug 2024 14:47:07 -0600 Subject: [PATCH 350/477] add comment about non-bfb with 0 --- .../eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index d53bf4935dbd..e947946fa203 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -160,7 +160,7 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] == 0); - REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 REQUIRE(SDS.shoc_ql[offset] >= 0); } } @@ -206,7 +206,7 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] != 0); REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); - REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 REQUIRE(SDS.shoc_ql[offset] >= 0); REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); } From 118865ff3a59f240efb4527dae5b6467449cf991 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 21 Aug 2024 11:19:28 -0600 Subject: [PATCH 351/477] Fix hist regex for scream The file names in the output testmods were producing files that were getting skipped by CIME due to not matching the regex in config_archive.xml. --- cime_config/config_archive.xml | 2 +- .../testdefs/testmods_dirs/scream/output/diags/shell_commands | 2 +- .../testdefs/testmods_dirs/scream/output/phys/shell_commands | 2 +- .../testmods_dirs/scream/output/phys_dyn/shell_commands | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cime_config/config_archive.xml b/cime_config/config_archive.xml index 84de6e0f171a..dee8d1992981 100644 --- a/cime_config/config_archive.xml +++ b/cime_config/config_archive.xml @@ -29,7 +29,7 @@ r\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* rhist\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* - hi\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d*\.\d{4}-\d{2}-\d{2}-\d{5}\.nc$ + .*\.h\..*\.nc$ diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands index 904f46b580f2..54bbf0aba648 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands @@ -29,7 +29,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.diags.hi \ + --prefix ${CASE}.scream.diags.h \ --grid "Physics ${PGTYPE}" \ --fields ${FIELDS} diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands index 116cdf111b43..e327f4d03365 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands @@ -26,7 +26,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.phys.hi \ + --prefix ${CASE}.scream.phys.h \ --grid "Physics ${PGTYPE}" \ --fields ${FIELDS} diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands index 0952b4afe3f9..c20a77d30d5c 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands @@ -26,7 +26,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.phys_dyn.hi \ + --prefix ${CASE}.scream.phys_dyn.h \ --grid Dynamics \ --io-grid 'Physics GLL' \ --fields ${FIELDS} From f6430aa5730b6baa6ad766fdf6fe75f3ee39bb8a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 10 Jul 2024 15:04:00 -0500 Subject: [PATCH 352/477] instance MOAB meshes on eamxx side need to change semoab_mod to accept pgN as a parameter --- .../homme/interface/phys_grid_mod.F90 | 51 +++++++++++++++++++ components/homme/src/share/semoab_mod.F90 | 5 +- .../homme/src/theta-l_kokkos/CMakeLists.txt | 1 + 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 index 4d928c800b03..5eaab1204e65 100644 --- a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 @@ -542,6 +542,14 @@ subroutine phys_grid_init (pgN) use gllfvremap_mod, only: gfr_init use homme_context_mod, only: elem, par use dimensions_mod, only: nelem, nelemd +#ifdef HAVE_MOAB + use seq_comm_mct, only: MHID, MHFID ! id of homme moab coarse and fine applications + use seq_comm_mct, only: ATMID + use seq_comm_mct, only: mhpgid ! id of pgx moab application + use semoab_mod, only: create_moab_meshes + use iMOAB, only : iMOAB_RegisterApplication + use iso_c_binding +#endif ! ! Input(s) ! @@ -554,6 +562,10 @@ subroutine phys_grid_init (pgN) character(2) :: str type(pg_specs_t), pointer :: pg +#ifdef HAVE_MOAB + integer :: ATM_ID1 + character*32 appname +#endif pg => pg_specs(pgN) if (pg%inited) then @@ -605,6 +617,45 @@ subroutine phys_grid_init (pgN) call compute_global_dofs (pg) call compute_global_coords (pg) call compute_global_area (pg) +#ifdef HAVE_MOAB + if (pgN > 0) then + appname="HM_COARSE"//C_NULL_CHAR + ATM_ID1 = 120 ! + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, MHID) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHID=", MHID + write(iulog,*) " " + endif + appname="HM_FINE"//C_NULL_CHAR + ATM_ID1 = 119 ! this number should not conflict with other components IDs; how do we know? + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, MHFID) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app for fine mesh') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHFID=", MHFID + write(iulog,*) " " + endif + appname="HM_PGX"//C_NULL_CHAR + ATM_ID1 = ATMID(1) ! this number should not conflict with other components IDs; how do we know? + ! + ! in this case, we reuse the main atm id, mhid will not be used for intersection anymore + ! still, need to be careful + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, mhpgid) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app for fine mesh') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHPGID=", mhpgid + write(iulog,*) " " + endif + + call create_moab_meshes(par, elem, pgN) + endif +#endif end subroutine phys_grid_init diff --git a/components/homme/src/share/semoab_mod.F90 b/components/homme/src/share/semoab_mod.F90 index 2d8bb099ade3..4841cb84b543 100644 --- a/components/homme/src/share/semoab_mod.F90 +++ b/components/homme/src/share/semoab_mod.F90 @@ -23,7 +23,7 @@ module semoab_mod use seq_comm_mct, only: MHPGID ! app id on moab side, for PGx style mesh, uniform from se use seq_comm_mct, only: atm_pg_active ! turn it on when PG style mesh active - use dyn_grid, only: fv_nphys ! phys grid mesh will be replicated too + !use dyn_grid, only: fv_nphys ! phys grid mesh will be replicated too use gllfvremap_mod, only: gfr_f_get_corner_latlon use control_mod, only : west, east, south, north ! 1, 2, 3, 4 @@ -76,7 +76,7 @@ integer function search_in(intarr, leng, value) end function search_in - subroutine create_moab_meshes(par, elem) + subroutine create_moab_meshes(par, elem, fv_nphys) use ISO_C_BINDING use iMOAB, only: iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & @@ -86,6 +86,7 @@ subroutine create_moab_meshes(par, elem) type (element_t), intent(inout) :: elem(:) type (parallel_t) , intent(in) :: par + integer , intent(in) :: fv_nphys integer ierr, i, j, ie, iv, block_ID, k, numvals integer icol, irow, je, linx ! local indices in fine el connect diff --git a/components/homme/src/theta-l_kokkos/CMakeLists.txt b/components/homme/src/theta-l_kokkos/CMakeLists.txt index 739866960e36..a585090ea76a 100644 --- a/components/homme/src/theta-l_kokkos/CMakeLists.txt +++ b/components/homme/src/theta-l_kokkos/CMakeLists.txt @@ -106,6 +106,7 @@ MACRO(THETAL_KOKKOS_SETUP) ${SRC_SHARE_DIR}/planar_mod.F90 ${SRC_SHARE_DIR}/geometry_mod.F90 ${SRC_SHARE_DIR}/planar_mesh_mod.F90 + ${SRC_SHARE_DIR}/semoab_mod.F90 ) IF (NOT HOMME_ENABLE_COMPOSE) From 7e04e52e4dba370899396e1bf239eb4a5e22905d Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 10 Jul 2024 23:04:52 -0500 Subject: [PATCH 353/477] phys atm model --- .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 171 +++++++++++++++++- 1 file changed, 170 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index dc57d828bbdb..b562dbce99db 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -17,6 +17,15 @@ module atm_comp_mct ! MPI use mpi +#ifdef HAVE_MOAB + use seq_comm_mct , only: mphaid ! atm physics grid id in MOAB, on atm pes + use iso_c_binding + use seq_comm_mct, only : num_moab_exports +#ifdef MOABCOMP + use seq_comm_mct, only: seq_comm_compare_mb_mct +#endif +#endif + implicit none save private @@ -107,7 +116,6 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) character(len=256) :: caseid, username, hostname character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c logical (kind=c_bool) :: restarted_run - !------------------------------------------------------------------------------- ! Grab some data from the cdata structure (coming from the coupler) @@ -178,6 +186,10 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Init MCT domain structure call atm_domain_mct (lsize, gsMap_atm, dom_atm) +#ifdef HAVE_MOAB + call moab_atm_phys_scream() +#endif + ! Init import/export mct attribute vectors call mct_aVect_init(x2a, rList=seq_flds_x2a_fields, lsize=lsize) call mct_aVect_init(a2x, rList=seq_flds_a2x_fields, lsize=lsize) @@ -356,4 +368,161 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) call mct_gGrid_importRAttr(dom_atm,"aream",data1,lsize) end subroutine atm_domain_mct +#ifdef HAVE_MOAB + subroutine moab_atm_phys_scream() + + use iMOAB, only : iMOAB_RegisterApplication, iMOAB_CreateVertices, iMOAB_WriteMesh, & + iMOAB_DefineTagStorage, iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_UpdateMeshInfo + use iso_c_binding, only: c_int, c_loc + use scream_f2c_mod, only: scream_get_num_global_cols, scream_get_num_local_cols, & + scream_get_local_cols_gids + use scream_f2c_mod, only: scream_get_cols_latlon, scream_get_cols_area + use seq_flds_mod, only: seq_flds_dom_fields + use shr_kind_mod , only: r8 => shr_kind_r8, cl=>shr_kind_cl, cxx=>shr_kind_cxx + use shr_const_mod, only: SHR_CONST_PI + use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields + + ! Local variables + ! + integer(kind=c_int) :: num_local_cols, num_global_cols + character*100 outfile, wopts + character(CXX) :: tagname ! will store all seq_flds_a2x_fields + character*32 appname + integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) + !------------------------------------------------------------------- + ! + ! Local Variables + ! + integer :: ierr,mpi_ierr ! error codes + integer :: i ! for loops along dofs + integer :: tagtype, tagindex, numco, ent_type + real(r8), pointer :: data1(:) + real(r8), pointer :: data2(:) ! temporary + integer(kind=c_int), allocatable, target :: col_gids(:) + real(r8), allocatable, target :: moab_vert_coords(:) + real(r8) :: latv, lonv + !------------------------------------------------------------------- + + appname="ATM_PHYS_SCREAM"//C_NULL_CHAR + ATM_PHYS = 200 + ATM_ID ! + ierr = iMOAB_RegisterApplication(appname, mpicom_atm, ATM_PHYS, mphaid) + if (ierr > 0 ) then + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + if (my_task == master_task) then + print *, " moab_atm_phys_scream:: register MOAB app:", trim(appname), " mphaid=", mphaid + endif + ! find global ids + num_local_cols = scream_get_num_local_cols() + num_global_cols = scream_get_num_global_cols() + + allocate(col_gids(num_local_cols)) + + call scream_get_local_cols_gids(c_loc(col_gids)) + + ! Fill in correct values for domain components + allocate(moab_vert_coords(num_local_cols*3)) + allocate( data1(num_local_cols), data2(num_local_cols) ) + call scream_get_cols_latlon(c_loc(data1),c_loc(data2)) + do i=1,num_local_cols + latv = data1(i) * SHR_CONST_PI/180 + lonv = data2(i) * SHR_CONST_PI/180 + moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*i )=SIN(latv) + enddo + ierr = iMOAB_CreateVertices(mphaid, num_local_cols*3, 3, moab_vert_coords) + if (ierr > 0 ) then + print *, "Error: fail to create MOAB vertices in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + tagtype = 0 ! dense, integer + numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define GLOBAL_ID tag in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ent_type = 0 ! vertex type + ierr = iMOAB_SetIntTagStorage(mphaid, tagname, num_local_cols , ent_type, col_gids) + if (ierr > 0 ) then + print *, "Error: fail to set GLOBAL_ID tag in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ierr = iMOAB_ResolveSharedEntities( mphaid, num_local_cols, col_gids) + if (ierr > 0 ) then + print *, "Error: fail to resolve shared ents in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + tagname=trim(seq_flds_dom_fields)//C_NULL_CHAR ! mask is double too lat:lon:hgt:area:aream:mask:frac + tagtype = 1 ! dense, double + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, 'Error: fail to create tags from seq_flds_dom_fields ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + call scream_get_cols_area(c_loc(data1)) + tagname = 'area'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set area ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + ! Mask and frac are both exactly 1 + data1 = 1.0 + tagname = 'mask'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set mask ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + tagname = 'frac'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set frac ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + !call mct_gGrid_importRAttr(dom_atm,"mask",data1,lsize) + !call mct_gGrid_importRAttr(dom_atm,"frac",data1,lsize) + + ! Aream is computed by mct, so give invalid initial value + data1 = -9999.0_R8 +#ifdef MOABDEBUG + outfile = 'AtmPhys.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) + if (ierr > 0 ) then + print *, "Error: fail to write PhysAtm mesh " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif +#endif + ! define fields seq_flds_a2x_fields + tagtype = 1 ! dense, double + numco = 1 ! one value per vertex / entity + tagname = trim(seq_flds_a2x_fields)//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define seq_flds_a2x_fields " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ! make sure this is defined too; it could have the same fields, but in different order, or really different + ! fields; need to make sure we have them + tagname = trim(seq_flds_x2a_fields)//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define seq_flds_x2a_fields " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + deallocate(col_gids) + deallocate(data1) + deallocate(data2) + end subroutine moab_atm_phys_scream +#endif + end module atm_comp_mct From b745f6b53ab1029503559dc218491620114e6042 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Thu, 11 Jul 2024 14:28:25 -0500 Subject: [PATCH 354/477] update mesh info is needed also, set aream to -9999.0, like mct it will be reset later in areacor --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index b562dbce99db..0cd903837197 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -457,6 +457,11 @@ subroutine moab_atm_phys_scream() print *, "Error: fail to resolve shared ents in phys atm model" call mpi_abort(mpicom_atm,ierr,mpi_ierr) endif + ierr = iMOAB_UpdateMeshInfo(mphaid) + if (ierr > 0 ) then + print *, "Error: fail to update mesh info in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif tagname=trim(seq_flds_dom_fields)//C_NULL_CHAR ! mask is double too lat:lon:hgt:area:aream:mask:frac tagtype = 1 ! dense, double ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) @@ -493,6 +498,13 @@ subroutine moab_atm_phys_scream() ! Aream is computed by mct, so give invalid initial value data1 = -9999.0_R8 + tagname = 'aream'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set aream ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + #ifdef MOABDEBUG outfile = 'AtmPhys.h5m'//C_NULL_CHAR wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR From b0446efa38313351e009d5b2c41f1fb3fd2c5ead Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 27 Jul 2024 15:01:45 -0500 Subject: [PATCH 355/477] initial fields to and from moab data structures modify the SCDataManager to add moab 2d fields duplicate just 2d fields, for devive and host, for import and export allocate a2x_am and x2a_am, and use extensively #ifdef HAVE_MOAB --- .../atmosphere_surface_coupling_exporter.cpp | 31 +++++ .../atmosphere_surface_coupling_exporter.hpp | 7 ++ .../atmosphere_surface_coupling_importer.cpp | 34 ++++++ .../atmosphere_surface_coupling_importer.hpp | 20 ++++ .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 111 ++++++++++++++++-- .../mct_coupling/scream_cxx_f90_interface.cpp | 18 ++- .../eamxx/src/mct_coupling/scream_f2c_mod.F90 | 15 ++- .../src/share/atm_process/SCDataManager.hpp | 15 ++- 8 files changed, 238 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp index 7a0253ec908a..9044df7e76cc 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp @@ -137,6 +137,13 @@ void SurfaceCouplingExporter::setup_surface_coupling_data(const SCDataManager &s m_num_cols, m_num_cpl_exports); m_cpl_exports_view_d = Kokkos::create_mirror_view(DefaultDevice(), m_cpl_exports_view_h); +#ifdef HAVE_MOAB + // The export data is of size num_cpl_exports,ncols. All other data is of size num_scream_exports + m_moab_cpl_exports_view_h = decltype(m_moab_cpl_exports_view_h) (sc_data_manager.get_field_data_moab_ptr(), + m_num_cpl_exports, m_num_cols); + m_moab_cpl_exports_view_d = Kokkos::create_mirror_view(DefaultDevice(), m_moab_cpl_exports_view_h); +#endif + m_export_field_names = new name_t[m_num_scream_exports]; std::memcpy(m_export_field_names, sc_data_manager.get_field_name_ptr(), m_num_scream_exports*32*sizeof(char)); @@ -536,6 +543,12 @@ void SurfaceCouplingExporter::do_export_to_cpl(const bool called_during_initiali // Any field not exported by scream, or not exported // during initialization, is set to 0.0 Kokkos::deep_copy(m_cpl_exports_view_d, 0.0); +#ifdef HAVE_MOAB + // Any field not exported by scream, or not exported + // during initialization, is set to 0.0 + Kokkos::deep_copy(m_moab_cpl_exports_view_d, 0.0); + const auto moab_cpl_exports_view_d = m_moab_cpl_exports_view_d; +#endif const auto cpl_exports_view_d = m_cpl_exports_view_d; const int num_exports = m_num_scream_exports; const int num_cols = m_num_cols; @@ -554,9 +567,27 @@ void SurfaceCouplingExporter::do_export_to_cpl(const bool called_during_initiali cpl_exports_view_d(icol,info.cpl_indx) = info.constant_multiple*info.data[offset]; } }); +#ifdef HAVE_MOAB + Kokkos::parallel_for(export_policy, KOKKOS_LAMBDA(const int& i) { + const int ifield = i / num_cols; + const int icol = i % num_cols; + const auto& info = col_info(ifield); + const auto offset = icol*info.col_stride + info.col_offset; + // if this is during initialization, check whether or not the field should be exported + bool do_export = (not called_during_initialization || info.transfer_during_initialization); + if (do_export) { + moab_cpl_exports_view_d(info.cpl_indx, icol) = info.constant_multiple*info.data[offset]; + } + }); +#endif // Deep copy fields from device to cpl host array Kokkos::deep_copy(m_cpl_exports_view_h,m_cpl_exports_view_d); +#ifdef HAVE_MOAB + // Deep copy fields from device to cpl host array + Kokkos::deep_copy(m_moab_cpl_exports_view_h,m_moab_cpl_exports_view_d); +#endif + } // ========================================================================================= void SurfaceCouplingExporter::finalize_impl() diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp index d8ccf5862f3c..ca7772fdf1be 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp @@ -141,6 +141,13 @@ class SurfaceCouplingExporter : public AtmosphereProcess view_2d m_cpl_exports_view_d; uview_2d m_cpl_exports_view_h; +#ifdef HAVE_MOAB + // Views storing a 2d array with dims (num_fields, num_cols) for moab cpl export data. + // The field cols strides faster, since that's what moab does (so we can "view" the + // pointer to the whole a2x_am(:,:) array from Fortran) + view_2d m_moab_cpl_exports_view_d; + uview_2d m_moab_cpl_exports_view_h; +#endif // Array storing the field names for exports name_t* m_export_field_names; std::vector m_export_field_names_vector; diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 7573993a3189..7686319ad3d8 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -74,6 +74,13 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptrget_params().get("iop_srf_prop")) { // Overwrite imports with data from IOP file @@ -232,6 +263,9 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in const auto& info_d = col_info_d(ifield); const auto offset = icol*info_d.col_stride + info_d.col_offset; info_d.data[offset] = col_val; +#ifdef HAVE_MOAB + // TODO +#endif }); } } diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp index 3a34a8b2951e..2bb0aa3ad4db 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp @@ -55,12 +55,24 @@ class SurfaceCouplingImporter : public AtmosphereProcess // which are not needed during initialization. void do_import(const bool called_during_initialization=false); +#ifdef HAVE_MOAB + // Function which performes the import from moab coupler to scream fields. + // If calling in initialize_impl(), set + // called_during_initialization=true to avoid importing to fields + // which are not needed during initialization. + void do_import_from_moab(const bool called_during_initialization=false); +#endif + // Take and store data from SCDataManager void setup_surface_coupling_data(const SCDataManager &sc_data_manager); // Overwrite imports for IOP cases with IOP file surface data void overwrite_iop_imports (const bool called_during_initialization); +#ifdef HAVE_MOAB + // Overwrite imports for IOP cases with IOP file surface data + void overwrite_iop_imports_from_moab (const bool called_during_initialization); +#endif protected: // The three main overrides for the subcomponent @@ -83,6 +95,14 @@ class SurfaceCouplingImporter : public AtmosphereProcess view_2d m_cpl_imports_view_d; uview_2d m_cpl_imports_view_h; +#ifdef HAVE_MOAB + // Views storing a 2d array with dims (num_fields,num_cols) for import data. + // The colums index strides faster, since that's what moab does (so we can "view" the + // pointer to the whole x2a_am(:,:) array from Fortran) + view_2d m_moab_cpl_imports_view_d; + uview_2d m_moab_cpl_imports_view_h; +#endif + // Array storing the field names for imports name_t* m_import_field_names; diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 0cd903837197..54f0338f0c66 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -20,7 +20,6 @@ module atm_comp_mct #ifdef HAVE_MOAB use seq_comm_mct , only: mphaid ! atm physics grid id in MOAB, on atm pes use iso_c_binding - use seq_comm_mct, only : num_moab_exports #ifdef MOABCOMP use seq_comm_mct, only: seq_comm_compare_mb_mct #endif @@ -54,6 +53,17 @@ module atm_comp_mct integer :: atm_log_unit + +#ifdef HAVE_MOAB + ! to store all fields to be set in moab + integer , private :: mblsize, totalmbls, nsend, totalmbls_r, nrecv + real(r8) , allocatable, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes + real(r8) , allocatable, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes +#ifdef MOABCOMP + integer :: mpicom_atm_moab ! used just for mpi-reducing the difference between moab tags and mct avs + integer :: rank2 +#endif +#endif !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -89,7 +99,13 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use seq_comm_mct, only: seq_comm_inst, seq_comm_name, seq_comm_suffix use shr_file_mod, only: shr_file_getunit, shr_file_setIO use shr_sys_mod, only: shr_sys_abort - +#ifdef HAVE_MOAB + use shr_kind_mod , only: cxx=>shr_kind_cxx + use iMOAB, only: iMOAB_SetDoubleTagStorage +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#endif ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) , intent(inout) :: EClock type(seq_cdata) , intent(inout) :: cdata @@ -109,7 +125,13 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) type(c_ptr) :: x2a_ptr, a2x_ptr character(len=256) :: atm_log_fname ! name of ATM log file character(CL) :: calendar ! calendar string - +#ifdef HAVE_MOAB + integer :: ent_type + character(CXX) :: tagname ! will store all seq_flds_a2x_fields +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif ! TODO: read this from the namelist? character(len=256) :: yaml_fname = "./data/scream_input.yaml" character(kind=c_char,len=256), target :: yaml_fname_c, atm_log_fname_c @@ -207,12 +229,33 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Init surface coupling stuff in the AD call scream_set_cpl_indices (x2a, a2x) +#ifdef HAVE_MOAB + mblsize = lsize + nsend = mct_avect_nRattr(a2x) + totalmbls = mblsize * nsend ! size of the double array + allocate (a2x_am(mblsize, nsend) ) + a2x_am = 0. ! initialize everything with 0 + + nrecv = mct_avect_nRattr(x2a) + totalmbls_r = mblsize * nrecv ! size of the double array used to receive + allocate (x2a_am(mblsize, nrecv) ) ! these will be received by moab tags, then used to set cam in surf data + x2a_am = 0. ! initialize everything with 0 +#endif + call scream_setup_surface_coupling (c_loc(import_field_names), c_loc(import_cpl_indices), & - c_loc(x2a%rAttr), c_loc(import_vector_components), & + c_loc(x2a%rAttr), & +#ifdef HAVE_MOAB + c_loc(x2a_am), & +#endif + c_loc(import_vector_components), & c_loc(import_constant_multiple), c_loc(do_import_during_init), & num_cpl_imports, num_scream_imports, import_field_size, & c_loc(export_field_names), c_loc(export_cpl_indices), & - c_loc(a2x%rAttr), c_loc(export_vector_components), & + c_loc(a2x%rAttr), & +#ifdef HAVE_MOAB + c_loc(a2x_am), & +#endif + c_loc(export_vector_components), & c_loc(export_constant_multiple), c_loc(do_export_during_init), & num_cpl_exports, num_scream_exports, export_field_size) @@ -220,6 +263,24 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(trim(username),username_c) call string_f2c(trim(hostname),hostname_c) call scream_init_atm (caseid_c,hostname_c,username_c) +#ifdef HAVE_MOAB + ! data should be set now inside moab from import and export fields + ! do we import and export or just export at init stage ? + tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, totalmbls , ent_type, a2x_am ) + if (ierr /= 0) then + print *,'moab error in setting data in a2x_am ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + end if +#ifdef MOABDEBUG + outfile = 'AtmPhys_init.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) + if (ierr > 0 ) & + call mpi_abort(mpicom_atm,ierr,mpi_ierr) +#endif +#endif end subroutine atm_init_mct @@ -227,6 +288,14 @@ end subroutine atm_init_mct subroutine atm_run_mct(EClock, cdata, x2a, a2x) use iso_c_binding, only: c_double use scream_f2c_mod, only: scream_run +#ifdef HAVE_MOAB + use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields + use shr_kind_mod , only: cxx=>shr_kind_cxx + use iMOAB, only: iMOAB_SetDoubleTagStorage, iMOAB_GetDoubleTagStorage +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#endif ! !INPUT/OUTPUT PARAMETERS: @@ -241,7 +310,14 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) type(mct_gGrid) , pointer :: ggrid real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream - +#ifdef HAVE_MOAB + integer :: ent_type + character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields +#ifdef MOABDEBUG + integer :: cur_atm_stepno, ierr + character*100 :: outfile, wopts, lnum +#endif +#endif !------------------------------------------------------------------------------- call seq_cdata_setptrs(cdata, & @@ -251,10 +327,29 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) ! Get time step info call seq_timemgr_EClockGetData (EClock, next_cday=nextsw_cday, dtime=dt_scream) - +#ifdef HAVE_MOAB + ! import data from moab data structures + tagname=trim(seq_flds_x2a_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_GetDoubleTagStorage ( mphaid, tagname, totalmbls_r , ent_type, x2a_am ) + +#endif ! Run scream call scream_run( dt_scream ) +#ifdef HAVE_MOAB + ! export data to moab data structures + tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, totalmbls , ent_type, a2x_am ) +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_atm_stepno ) + write(lnum,"(I0.2)")cur_atm_stepno + outfile = 'AtmPhys_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) +#endif +#endif ! Set time of next radiadtion computation call seq_infodata_PutData(infodata, nextsw_cday=nextsw_cday) @@ -386,7 +481,9 @@ subroutine moab_atm_phys_scream() ! Local variables ! integer(kind=c_int) :: num_local_cols, num_global_cols +#ifdef MOABDEBUG character*100 outfile, wopts +#endif character(CXX) :: tagname ! will store all seq_flds_a2x_fields character*32 appname integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index e9129e6fcea3..4b630614c941 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -165,11 +165,19 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, } void scream_setup_surface_coupling (const char*& import_field_names, int*& import_cpl_indices, - double*& x2a_ptr, int*& import_vector_components, + double*& x2a_ptr, +#ifdef HAVE_MOAB + double*& x2a_moab_ptr, +#endif + int*& import_vector_components, double*& import_constant_multiple, bool*& do_import_during_init, const int& num_cpl_imports, const int& num_scream_imports, const int& import_field_size, char*& export_field_names, int*& export_cpl_indices, - double*& a2x_ptr, int*& export_vector_components, + double*& a2x_ptr, +#ifdef HAVE_MOAB + double*& a2x_moab_ptr, +#endif + int*& export_vector_components, double*& export_constant_multiple, bool*& do_export_during_init, const int& num_cpl_exports, const int& num_scream_exports, const int& export_field_size) { @@ -197,10 +205,16 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Import, num_cpl_imports, num_scream_imports, import_field_size, x2a_ptr, +#ifdef HAVE_MOAB + x2a_moab_ptr, +#endif names_in[0], import_cpl_indices, import_vector_components, import_constant_multiple, do_import_during_init); ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Export, num_cpl_exports, num_scream_exports, export_field_size, a2x_ptr, +#ifdef HAVE_MOAB + x2a_moab_ptr, +#endif names_out[0], export_cpl_indices, export_vector_components, export_constant_multiple, do_export_during_init); }); diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index f7a0bc9c3794..5e09def98df0 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -47,11 +47,19 @@ end subroutine scream_get_cols_area ! responsible to handle import/export operation from/into the component ! coupler surface fluxes/state structures subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices, & - x2a_ptr, import_vector_components, & + x2a_ptr, & +#ifdef HAVE_MOAB + x2a_moab_ptr, & +#endif + import_vector_components, & import_constant_multiple, do_import_during_init, & num_cpl_imports, num_scream_imports, import_field_size, & export_field_names, export_cpl_indices, & - a2x_ptr, export_vector_components, & + a2x_ptr, & +#ifdef HAVE_MOAB + a2x_moab_ptr, & +#endif + export_vector_components, & export_constant_multiple, do_export_during_init, & num_cpl_exports, num_scream_exports, export_field_size) bind(c) use iso_c_binding, only: c_ptr, c_int @@ -64,6 +72,9 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices type(c_ptr), intent(in) :: export_field_names, export_cpl_indices, & a2x_ptr, export_vector_components, & export_constant_multiple, do_export_during_init +#ifdef HAVE_MOAB + type(c_ptr), intent(in) :: x2a_moab_ptr, a2x_moab_ptr +#endif integer(kind=c_int), intent(in) :: num_cpl_imports, num_scream_imports, & num_cpl_exports, num_scream_exports integer(kind=c_int), intent(in) :: import_field_size, export_field_size diff --git a/components/eamxx/src/share/atm_process/SCDataManager.hpp b/components/eamxx/src/share/atm_process/SCDataManager.hpp index 02af15423e62..51384750d435 100644 --- a/components/eamxx/src/share/atm_process/SCDataManager.hpp +++ b/components/eamxx/src/share/atm_process/SCDataManager.hpp @@ -22,7 +22,11 @@ struct SCDataManager { ~SCDataManager() = default; void setup_internals (const int num_cpl_fields, const int num_scream_fields, const int field_size, - Real* field_data_ptr, char* field_names, int* field_cpl_indices_ptr, + Real* field_data_ptr, +#ifdef HAVE_MOAB + Real* field_data_moab_ptr, +#endif + char* field_names, int* field_cpl_indices_ptr, int* field_vector_components_ptr, Real* field_constant_multiple_ptr, bool* transfer_during_init_ptr) { @@ -38,6 +42,9 @@ struct SCDataManager { EKAT_ASSERT_MSG(field_constant_multiple_ptr !=nullptr, "Error! Ptr for constant multiple is null."); EKAT_ASSERT_MSG(transfer_during_init_ptr !=nullptr, "Error! Ptr for initial transfer boolean is null."); m_field_data = decltype(m_field_data) (field_data_ptr, m_field_size, m_num_cpl_fields); +#ifdef HAVE_MOAB + m_field_data_moab = decltype(m_field_data_moab) (field_data_moab_ptr, m_num_cpl_fields, m_field_size); +#endif m_field_cpl_indices = decltype(m_field_cpl_indices) (field_cpl_indices_ptr, m_num_scream_fields); m_field_vector_components = decltype(m_field_vector_components) (field_vector_components_ptr, m_num_scream_fields); m_field_constant_multiple = decltype(m_field_constant_multiple) (field_constant_multiple_ptr, m_num_scream_fields); @@ -65,7 +72,11 @@ struct SCDataManager { Real* get_field_data_ptr () const { return m_field_data.data(); } - +#ifdef HAVE_MOAB + Real* get_field_data_moab_ptr () const { + return m_field_data_moab.data(); + } +#endif Real get_field_data_view_entry(const int i, const int f) { return m_field_data(i, f); } From d881e0486f082e89f8eb1392c2e0df7ea7cdd71c Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 27 Jul 2024 15:49:37 -0500 Subject: [PATCH 356/477] setup_internals moab changes --- components/eamxx/src/control/atmosphere_driver.cpp | 6 ++++++ components/eamxx/src/control/atmosphere_driver.hpp | 3 +++ components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 5 +++-- components/eamxx/src/share/atm_process/SCDataManager.hpp | 3 +++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index c3da1bacd3f0..211b75368981 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -300,6 +300,9 @@ void AtmosphereDriver::create_grids() void AtmosphereDriver::setup_surface_coupling_data_manager(SurfaceCouplingTransferType transfer_type, const int num_cpl_fields, const int num_scream_fields, const int field_size, Real* data_ptr, +#ifdef HAVE_MOAB + Real* data_ptr_moab, +#endif char* names_ptr, int* cpl_indices_ptr, int* vec_comps_ptr, Real* constant_multiple_ptr, bool* do_transfer_during_init_ptr) { @@ -318,6 +321,9 @@ void AtmosphereDriver::setup_surface_coupling_data_manager(SurfaceCouplingTransf } else EKAT_ERROR_MSG("Error! Unexpected SurfaceCouplingTransferType."); sc_data_mgr->setup_internals(num_cpl_fields, num_scream_fields, field_size, data_ptr, +#ifdef HAVE_MOAB + data_ptr_moab, +#endif names_ptr, cpl_indices_ptr, vec_comps_ptr, constant_multiple_ptr, do_transfer_during_init_ptr); } diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index d43d233f4e97..7e96225210db 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -87,6 +87,9 @@ class AtmosphereDriver void setup_surface_coupling_data_manager(SurfaceCouplingTransferType transfer_type, const int num_cpl_fields, const int num_scream_fields, const int field_size, Real* data_ptr, +#ifdef HAVE_MOAB + Real* data_ptr_moab, +#endif char* names_ptr, int* cpl_indices_ptr, int* vec_comps_ptr, Real* constant_multiple_ptr, bool* do_transfer_during_init_ptr); diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 54f0338f0c66..e9f3397f827b 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -57,8 +57,8 @@ module atm_comp_mct #ifdef HAVE_MOAB ! to store all fields to be set in moab integer , private :: mblsize, totalmbls, nsend, totalmbls_r, nrecv - real(r8) , allocatable, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes - real(r8) , allocatable, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes + real(r8) , allocatable, target, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes + real(r8) , allocatable, target, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes #ifdef MOABCOMP integer :: mpicom_atm_moab ! used just for mpi-reducing the difference between moab tags and mct avs integer :: rank2 @@ -101,6 +101,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use shr_sys_mod, only: shr_sys_abort #ifdef HAVE_MOAB use shr_kind_mod , only: cxx=>shr_kind_cxx + use mct_mod, only: mct_avect_nRattr use iMOAB, only: iMOAB_SetDoubleTagStorage #ifdef MOABDEBUG use iMOAB, only: iMOAB_WriteMesh diff --git a/components/eamxx/src/share/atm_process/SCDataManager.hpp b/components/eamxx/src/share/atm_process/SCDataManager.hpp index 51384750d435..2dcfd3c5016c 100644 --- a/components/eamxx/src/share/atm_process/SCDataManager.hpp +++ b/components/eamxx/src/share/atm_process/SCDataManager.hpp @@ -116,6 +116,9 @@ struct SCDataManager { int m_num_scream_fields; view_2d m_field_data; +#ifdef HAVE_MOAB + view_2d m_field_data_moab; +#endif name_t* m_field_names; view_1d m_field_cpl_indices; view_1d m_field_vector_components; From 9e87950a0604fb652b492c16de0f006592c52b7a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 27 Jul 2024 16:29:12 -0500 Subject: [PATCH 357/477] type, x2a instead of a2x --- components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 4b630614c941..52e0d692e0a5 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -213,7 +213,7 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Export, num_cpl_exports, num_scream_exports, export_field_size, a2x_ptr, #ifdef HAVE_MOAB - x2a_moab_ptr, + a2x_moab_ptr, #endif names_out[0], export_cpl_indices, export_vector_components, export_constant_multiple, do_export_during_init); From 8f0e75a01f65e75fdc0544fb3b030a1b85e79c1b Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 27 Jul 2024 21:48:55 -0500 Subject: [PATCH 358/477] remove unused moab import functions --- .../control/atmosphere_surface_coupling_importer.hpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp index 2bb0aa3ad4db..6cf04c4166d2 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp @@ -55,24 +55,12 @@ class SurfaceCouplingImporter : public AtmosphereProcess // which are not needed during initialization. void do_import(const bool called_during_initialization=false); -#ifdef HAVE_MOAB - // Function which performes the import from moab coupler to scream fields. - // If calling in initialize_impl(), set - // called_during_initialization=true to avoid importing to fields - // which are not needed during initialization. - void do_import_from_moab(const bool called_during_initialization=false); -#endif - // Take and store data from SCDataManager void setup_surface_coupling_data(const SCDataManager &sc_data_manager); // Overwrite imports for IOP cases with IOP file surface data void overwrite_iop_imports (const bool called_during_initialization); -#ifdef HAVE_MOAB - // Overwrite imports for IOP cases with IOP file surface data - void overwrite_iop_imports_from_moab (const bool called_during_initialization); -#endif protected: // The three main overrides for the subcomponent From ff21995779dc890737461850a1884d3caf01473f Mon Sep 17 00:00:00 2001 From: Vijay Mahadevan Date: Tue, 30 Jul 2024 14:12:19 -0500 Subject: [PATCH 359/477] Fix a build error when MOABDEBUG is undefined --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index e9f3397f827b..0a0a53007bde 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -312,10 +312,10 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream #ifdef HAVE_MOAB - integer :: ent_type + integer :: ent_type, ierr character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields #ifdef MOABDEBUG - integer :: cur_atm_stepno, ierr + integer :: cur_atm_stepno character*100 :: outfile, wopts, lnum #endif #endif From 07dc9f04134517fee735fbee158bf1862f26b75e Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 13 Aug 2024 16:49:40 -0500 Subject: [PATCH 360/477] add comparison logic (MOABCOMP) for atm imported fields --- .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 0a0a53007bde..96b243c7aabb 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -211,6 +211,10 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) #ifdef HAVE_MOAB call moab_atm_phys_scream() +#ifdef MOABCOMP + mpicom_atm_moab = mpicom_atm ! just to store it to be used later in MOABCOMP + rank2 = my_task ! again, just to use it for MOABCOMP output +#endif #endif ! Init import/export mct attribute vectors @@ -296,6 +300,16 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) #ifdef MOABDEBUG use iMOAB, only: iMOAB_WriteMesh #endif +#ifdef MOABCOMP + use mct_mod, only: mct_list, mct_string, mct_list_nitem, mct_string_toChar + use seq_comm_mct, only : num_moab_exports + + real(r8) :: difference + type(mct_list) :: temp_list + integer :: size_list, index_list, ent_type + type(mct_string) :: mctOStr ! + character(CXX) :: mct_field, modelStr +#endif #endif ! !INPUT/OUTPUT PARAMETERS: @@ -312,7 +326,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream #ifdef HAVE_MOAB - integer :: ent_type, ierr + integer :: ierr character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields #ifdef MOABDEBUG integer :: cur_atm_stepno @@ -329,6 +343,24 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) ! Get time step info call seq_timemgr_EClockGetData (EClock, next_cday=nextsw_cday, dtime=dt_scream) #ifdef HAVE_MOAB + +#ifdef MOABCOMP + ! loop over all fields in seq_flds_x2a_fields + call mct_list_init(temp_list ,seq_flds_x2a_fields) + size_list=mct_list_nitem (temp_list) + ent_type = 0 ! entity type is vertex for phys atm + if (rank2 .eq. 0) print *, num_moab_exports, trim(seq_flds_x2a_fields) + modelStr ='atm run' + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call seq_comm_compare_mb_mct(modelStr, mpicom_atm_moab, x2a, mct_field, mphaid, tagname, ent_type, difference) + enddo + call mct_list_clean(temp_list) +#endif + + ! import data from moab data structures tagname=trim(seq_flds_x2a_fields)//C_NULL_CHAR ent_type = 0 ! vertices, point cloud From 00214222b49a8d84f51a7b977c665af935944d39 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Tue, 13 Aug 2024 17:12:13 -0500 Subject: [PATCH 361/477] compile error --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 96b243c7aabb..b5513afb4cca 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -301,7 +301,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) use iMOAB, only: iMOAB_WriteMesh #endif #ifdef MOABCOMP - use mct_mod, only: mct_list, mct_string, mct_list_nitem, mct_string_toChar + use mct_mod use seq_comm_mct, only : num_moab_exports real(r8) :: difference From c1f6113b1a441b6e01aeca44effc6e2741578e78 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 21 Aug 2024 16:37:26 -0500 Subject: [PATCH 362/477] new api in imoab to apply weights apply manually the commit 3bea55f109ccdca502d26909fbb0a66054a16c4b from the E3SM repo, not available yet in scream repo, because it is not synchronized to master E3SM yet I mean, the last sync was after we did merged 3bea55f109ccd to E3SM master --- driver-moab/main/seq_map_mod.F90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/driver-moab/main/seq_map_mod.F90 b/driver-moab/main/seq_map_mod.F90 index be47473f444c..82e67b8b195e 100644 --- a/driver-moab/main/seq_map_mod.F90 +++ b/driver-moab/main/seq_map_mod.F90 @@ -355,6 +355,7 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, real(r8), dimension(:), allocatable :: wghts real(kind=r8) , allocatable :: targtags(:,:), targtags_ini(:,:) real(kind=r8) :: factor + integer :: filter_type ! used for caas projection #endif ! ! Local Variables @@ -649,7 +650,8 @@ subroutine seq_map_map( mapper, av_s, av_d, fldlist, norm, avwts_s, avwtsfld_s, call shr_sys_flush(logunit) endif #endif - ierr = iMOAB_ApplyScalarProjectionWeights ( mapper%intx_mbid, mapper%weight_identifier, fldlist_moab, fldlist_moab) + filter_type = 0 ! no + ierr = iMOAB_ApplyScalarProjectionWeights ( mapper%intx_mbid, filter_type, mapper%weight_identifier, fldlist_moab, fldlist_moab) if (ierr .ne. 0) then write(logunit,*) subname,' error in applying weights ' call shr_sys_abort(subname//' ERROR in applying weights') From 251431904a6610f2a664b317c035f27ffd44a76c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 22 Aug 2024 15:02:09 -0600 Subject: [PATCH 363/477] Fix regex. Use lookahead to avoid restart files --- cime_config/config_archive.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/config_archive.xml b/cime_config/config_archive.xml index dee8d1992981..16320e8a93a4 100644 --- a/cime_config/config_archive.xml +++ b/cime_config/config_archive.xml @@ -29,7 +29,7 @@ r\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* rhist\.(INSTANT|AVERAGE|MAX|MIN)\.n(step|sec|min|hour|day|month|year)s_x\d* - .*\.h\..*\.nc$ + .*\.h\.(?!rhist\.).*\.nc$ From c8679fd45253d4896c0f61d384c44e4c36b334b1 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 22 Aug 2024 20:08:23 -0700 Subject: [PATCH 364/477] options to build sk p3 and shoc separately --- components/eamxx/CMakeLists.txt | 6 ++-- .../eamxx/src/physics/p3/CMakeLists.txt | 6 ++-- .../src/physics/p3/impl/p3_main_impl.hpp | 2 +- .../eamxx/src/physics/p3/p3_functions.hpp | 20 +++++------ .../eamxx/src/physics/p3/tests/CMakeLists.txt | 2 +- .../eamxx/src/physics/shoc/CMakeLists.txt | 6 ++-- .../shoc/eamxx_shoc_process_interface.cpp | 10 +++--- .../shoc/eamxx_shoc_process_interface.hpp | 10 +++--- .../src/physics/shoc/impl/shoc_main_impl.hpp | 6 ++-- .../eamxx/src/physics/shoc/shoc_functions.hpp | 36 +++++++++---------- .../src/physics/shoc/shoc_functions_f90.cpp | 4 +-- .../src/physics/shoc/tests/CMakeLists.txt | 2 +- components/eamxx/src/scream_config.h.in | 4 ++- 13 files changed, 59 insertions(+), 55 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 398efa3bc2df..a60fc1e342fe 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -198,8 +198,10 @@ set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C inst set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) -set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels") -if (NOT SCREAM_SMALL_KERNELS) +set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for all components") +set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") +set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") +if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") endif() diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 1a01f607c630..fc0293cb35d0 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -74,14 +74,14 @@ set(P3_SK_SRCS ) set(P3_LIBS "p3") -if (SCREAM_SMALL_KERNELS) +if (SCREAM_P3_SMALL_KERNELS) add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) else() add_library(p3 ${P3_SRCS}) if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) - # Always build shoc_sk with SCREAM_SMALL_KERNELS on - target_compile_definitions(p3_sk PUBLIC "SCREAM_SMALL_KERNELS") + # Always build p3_sk with SCREAM_P3_SMALL_KERNELS on + target_compile_definitions(p3_sk PUBLIC "SCREAM_P3_SMALL_KERNELS") list(APPEND P3_LIBS "p3_sk") endif() endif() diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index ce5116cab2fa..c98a37d0557a 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -349,7 +349,7 @@ ::p3_main( Int nk, const physics::P3_Constants & p3constants) { -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_P3_SMALL_KERNELS return p3_main_internal(runtime_options, prognostic_state, diagnostic_inputs, diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 47bb4560c2e1..3a5218f5c001 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -420,7 +420,7 @@ struct Functions const uview_1d& nc_tend, Scalar& precip_liq_surf); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void cloud_sedimentation_disp( const uview_2d& qc_incld, const uview_2d& rho, @@ -468,7 +468,7 @@ struct Functions Scalar& precip_liq_surf, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void rain_sedimentation_disp( const uview_2d& rho, const uview_2d& inv_rho, @@ -518,7 +518,7 @@ struct Functions Scalar& precip_ice_surf, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void ice_sedimentation_disp( const uview_2d& rho, const uview_2d& inv_rho, @@ -562,7 +562,7 @@ struct Functions const uview_1d& bm, const uview_1d& th_atm); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void homogeneous_freezing_disp( const uview_2d& T_atm, const uview_2d& inv_exner, @@ -868,7 +868,7 @@ struct Functions const Int& timestepcount, const bool& force_abort, const Int& source_ind, const MemberType& team, const uview_1d& col_loc); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void check_values_disp(const uview_2d& qv, const uview_2d& temp, const Int& ktop, const Int& kbot, const Int& timestepcount, const bool& force_abort, const Int& source_ind, const uview_2d& col_loc, const Int& nj, const Int& nk); @@ -914,7 +914,7 @@ struct Functions Scalar& precip_ice_surf, view_1d_ptr_array& zero_init); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_init_disp( const Int& nj,const Int& nk_pack, const uview_2d& cld_frac_i, const uview_2d& cld_frac_l, @@ -988,7 +988,7 @@ struct Functions bool& is_hydromet_present, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part1_disp( const Int& nj, const Int& nk, @@ -1121,7 +1121,7 @@ struct Functions const Int& nk, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part2_disp( const Int& nj, const Int& nk, @@ -1246,7 +1246,7 @@ struct Functions const uview_1d& diag_eff_radius_qr, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part3_disp( const Int& nj, const Int& nk_pack, @@ -1319,7 +1319,7 @@ struct Functions Int nk, // number of vertical cells per column const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static Int p3_main_internal_disp( const P3Runtime& runtime_options, const P3PrognosticState& prognostic_state, diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 4e8901069e05..912c7cedac66 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -60,7 +60,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) LABELS "p3;physics;fail" ${FORCE_RUN_DIFF_FAILS}) - if (NOT SCREAM_SMALL_KERNELS) + if (NOT SCREAM_P3_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 891626b9d3df..4796b30abefd 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -104,14 +104,14 @@ if (HIP_BUILD) endif() set(SHOC_LIBS "shoc") -if (SCREAM_SMALL_KERNELS) +if (SCREAM_SHOC_SMALL_KERNELS) add_library(shoc ${SHOC_SRCS} ${SHOC_SK_SRCS}) else() add_library(shoc ${SHOC_SRCS}) if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(shoc_sk ${SHOC_SRCS} ${SHOC_SK_SRCS}) - # Always build shoc_sk with SCREAM_SMALL_KERNELS on - target_compile_definitions(shoc_sk PUBLIC "SCREAM_SMALL_KERNELS") + # Always build shoc_sk with SCREAM_SHOC_SMALL_KERNELS on + target_compile_definitions(shoc_sk PUBLIC "SCREAM_SHOC_SMALL_KERNELS") list(APPEND SHOC_LIBS "shoc_sk") endif() endif() diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 657f9487611c..32bcd9bde0e2 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -156,7 +156,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) using scalar_view_t = decltype(m_buffer.wpthlp_sfc); scalar_view_t* _1d_scalar_view_ptrs[Buffer::num_1d_scalar_ncol] = {&m_buffer.wpthlp_sfc, &m_buffer.wprtp_sfc, &m_buffer.upwp_sfc, &m_buffer.vpwp_sfc -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.se_b, &m_buffer.ke_b, &m_buffer.wv_b, &m_buffer.wl_b , &m_buffer.se_a, &m_buffer.ke_a, &m_buffer.wv_a, &m_buffer.wl_a , &m_buffer.kbfs, &m_buffer.ustar2, &m_buffer.wstar @@ -182,7 +182,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.z_mid, &m_buffer.rrho, &m_buffer.thv, &m_buffer.dz, &m_buffer.zt_grid, &m_buffer.wm_zt, &m_buffer.inv_exner, &m_buffer.thlm, &m_buffer.qw, &m_buffer.dse, &m_buffer.tke_copy, &m_buffer.qc_copy, &m_buffer.shoc_ql2, &m_buffer.shoc_mix, &m_buffer.isotropy, &m_buffer.w_sec, &m_buffer.wqls_sec, &m_buffer.brunt -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.tabs, &m_buffer.dz_zt #endif }; @@ -191,7 +191,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.z_int, &m_buffer.rrho_i, &m_buffer.zi_grid, &m_buffer.thl_sec, &m_buffer.qw_sec, &m_buffer.qwthl_sec, &m_buffer.wthl_sec, &m_buffer.wqw_sec, &m_buffer.wtke_sec, &m_buffer.uw_sec, &m_buffer.vw_sec, &m_buffer.w3 -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.dz_zi #endif }; @@ -353,7 +353,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) history_output.wqls_sec = m_buffer.wqls_sec; history_output.brunt = m_buffer.brunt; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS temporaries.se_b = m_buffer.se_b; temporaries.ke_b = m_buffer.ke_b; temporaries.wv_b = m_buffer.wv_b; @@ -485,7 +485,7 @@ void SHOCMacrophysics::run_impl (const double dt) // Run shoc main SHF::shoc_main(m_num_cols, m_num_levs, m_num_levs+1, m_npbl, m_nadv, m_num_tracers, dt, workspace_mgr,runtime_options,input,input_output,output,history_output -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , temporaries #endif ); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 469c31de7af5..4d76efd4558b 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -382,13 +382,13 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Structure for storing local variables initialized using the ATMBufferManager struct Buffer { -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS static constexpr int num_1d_scalar_ncol = 4; #else static constexpr int num_1d_scalar_ncol = 15; #endif static constexpr int num_1d_scalar_nlev = 1; -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS static constexpr int num_2d_vector_mid = 18; static constexpr int num_2d_vector_int = 12; #else @@ -401,7 +401,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_1d wprtp_sfc; uview_1d upwp_sfc; uview_1d vpwp_sfc; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS uview_1d se_b; uview_1d ke_b; uview_1d wv_b; @@ -448,7 +448,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_2d w3; uview_2d wqls_sec; uview_2d brunt; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS uview_2d rho_zt; uview_2d shoc_qv; uview_2d tabs; @@ -506,7 +506,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess SHF::SHOCOutput output; SHF::SHOCHistoryOutput history_output; SHF::SHOCRuntime runtime_options; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS SHF::SHOCTemporaries temporaries; #endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 11383e772454..a41868478b8f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -61,7 +61,7 @@ Int Functions::shoc_init( return host_view(0); } -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS template KOKKOS_FUNCTION void Functions::shoc_main_internal( @@ -586,7 +586,7 @@ Int Functions::shoc_main( const SHOCInputOutput& shoc_input_output, // Input/Output const SHOCOutput& shoc_output, // Output const SHOCHistoryOutput& shoc_history_output // Output (diagnostic) -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , const SHOCTemporaries& shoc_temporaries // Temporaries for small kernels #endif ) @@ -608,7 +608,7 @@ Int Functions::shoc_main( const Scalar Ckh = shoc_runtime.Ckh; const Scalar Ckm = shoc_runtime.Ckm; -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS using ExeSpace = typename KT::ExeSpace; // SHOC main loop diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index d1d547245b1f..8913ad53841b 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -203,7 +203,7 @@ struct Functions view_2d isotropy; }; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS struct SHOCTemporaries { SHOCTemporaries() = default; @@ -272,7 +272,7 @@ struct Functions const uview_1d& zt_grid, const Scalar& phis, const uview_1d& host_dse); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void update_host_dse_disp( const Int& shcol, const Int& nlev, @@ -324,7 +324,7 @@ struct Functions const MemberType& team, const Int& nlev, const uview_1d& tke); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void check_tke_disp( const Int& schol, const Int& nlev, @@ -363,7 +363,7 @@ struct Functions Scalar& ke_int, Scalar& wv_int, Scalar& wl_int); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_energy_integrals_disp( const Int& shcol, const Int& nlev, @@ -409,7 +409,7 @@ struct Functions const Workspace& workspace, const uview_1d& thl_sec, const uview_1d& qw_sec, const uview_1d& wthl_sec, const uview_1d& wqw_sec, const uview_1d& qwthl_sec, const uview_1d& uw_sec, const uview_1d& vw_sec, const uview_1d& wtke_sec, const uview_1d& w_sec); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void diag_second_shoc_moments_disp( const Int& shcol, const Int& nlev, const Int& nlevi, const Scalar& thl2tune, @@ -485,7 +485,7 @@ struct Functions Scalar& ustar, Scalar& kbfs, Scalar& obklen); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_diag_obklen_disp( const Int& shcol, const Int& nlev, @@ -522,7 +522,7 @@ struct Functions const Workspace& workspace, const uview_1d& brunt, const uview_1d& shoc_mix); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_length_disp( const Int& shcol, const Int& nlev, @@ -564,7 +564,7 @@ struct Functions const uview_1d& pint, const Workspace& workspace, const uview_1d& host_dse); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_energy_fixer_disp( const Int& shcol, const Int& nlev, @@ -597,7 +597,7 @@ struct Functions const uview_1d& qw, const uview_1d& ql, const uview_1d& qv); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void compute_shoc_vapor_disp( const Int& shcol, const Int& nlev, @@ -614,7 +614,7 @@ struct Functions const uview_1d& ql, const uview_1d& inv_exner, const uview_1d& tabs); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void compute_shoc_temperature_disp( const Int& shcol, const Int& nlev, @@ -650,7 +650,7 @@ struct Functions const uview_1d& tke, const uview_1d& u_wind, const uview_1d& v_wind); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void update_prognostics_implicit_disp( const Int& shcol, const Int& nlev, @@ -697,7 +697,7 @@ struct Functions const uview_1d& zi_grid, const Workspace& workspace, const uview_1d& w3); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void diag_third_shoc_moments_disp( const Int& shcol, const Int& nlev, @@ -754,7 +754,7 @@ struct Functions const uview_1d& wqls, const uview_1d& wthv_sec, const uview_1d& shoc_ql2); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_assumed_pdf_disp( const Int& shcol, const Int& nlev, @@ -983,7 +983,7 @@ struct Functions const Int& ntop_shoc, const view_1d& pref_mid); -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS KOKKOS_FUNCTION static void shoc_main_internal( const MemberType& team, @@ -1166,7 +1166,7 @@ struct Functions const SHOCInputOutput& shoc_input_output, // Input/Output const SHOCOutput& shoc_output, // Output const SHOCHistoryOutput& shoc_history_output // Output (diagnostic) -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , const SHOCTemporaries& shoc_temporaries // Temporaries for small kernels #endif ); @@ -1237,7 +1237,7 @@ struct Functions const uview_1d& cldn, const Workspace& workspace, Scalar& pblh); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void pblintd_disp( const Int& shcol, const Int& nlev, @@ -1269,7 +1269,7 @@ struct Functions const uview_1d& dz_zt, const uview_1d& dz_zi, const uview_1d& rho_zt); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_grid_disp( const Int& shcol, const Int& nlev, @@ -1327,7 +1327,7 @@ struct Functions const uview_1d& tk, const uview_1d& tkh, const uview_1d& isotropy); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_tke_disp( const Int& shcol, const Int& nlev, diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index b5a2a00d27c8..e7468062d3c5 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2889,7 +2889,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, const auto nlevi_packs = ekat::npack(nlevi); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS view_1d se_b ("se_b", shcol), ke_b ("ke_b", shcol), @@ -2923,7 +2923,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, const auto elapsed_microsec = SHF::shoc_main(shcol, nlev, nlevi, npbl, nadv, num_qtracers, dtime, workspace_mgr, shoc_runtime_options, shoc_input, shoc_input_output, shoc_output, shoc_history_output -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , shoc_temporaries #endif ); diff --git a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt index 87e55959c528..f61de7d76a4c 100644 --- a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt @@ -78,7 +78,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} ) - if (NOT SCREAM_SMALL_KERNELS) + if (NOT SCREAM_SHOC_SMALL_KERNELS) CreateUnitTest(shoc_sk_tests "${SHOC_TESTS_SRCS}" LIBS shoc_sk THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} diff --git a/components/eamxx/src/scream_config.h.in b/components/eamxx/src/scream_config.h.in index 77b1bf04ff0c..a7c0f911f2e3 100644 --- a/components/eamxx/src/scream_config.h.in +++ b/components/eamxx/src/scream_config.h.in @@ -45,8 +45,10 @@ // Whether or not to run RRTMGP debug checks #cmakedefine SCREAM_RRTMGP_DEBUG -// Whether monolithic kernels are on +// Whether or not small kernels are on #cmakedefine SCREAM_SMALL_KERNELS +#cmakedefine SCREAM_P3_SMALL_KERNELS +#cmakedefine SCREAM_SHOC_SMALL_KERNELS // The sha of the last commit #define EAMXX_GIT_VERSION "${EAMXX_GIT_VERSION}" From c5bf2a3b07524d385c9b7cfe9a58ab21a6500eef Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 22 Aug 2024 20:40:07 -0700 Subject: [PATCH 365/477] add tests for small kernels --- cime_config/tests.py | 1 + .../testmods_dirs/scream/small_kernels_p3/shell_commands | 2 ++ .../testmods_dirs/scream/small_kernels_shoc/shell_commands | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands diff --git a/cime_config/tests.py b/cime_config/tests.py index ca3dba503c42..c293e611077e 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -650,6 +650,7 @@ "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.small_kernels--scream-output-preset-5", ) }, diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands new file mode 100644 index 000000000000..ea0e647fdcd3 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands @@ -0,0 +1,2 @@ +./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_P3_SMALL_KERNELS On' +$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands new file mode 100644 index 000000000000..da0429652975 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands @@ -0,0 +1,2 @@ +./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SHOC_SMALL_KERNELS On' +$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b From 81f66f59170a1505f8c793fd6507d8d640075d64 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 23 Aug 2024 00:26:04 -0400 Subject: [PATCH 366/477] use kokkos norm min v instead of std stuff --- .../src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp index 70fb386055a1..5794e21456f7 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp @@ -45,7 +45,7 @@ void Functions::shoc_assumed_pdf_compute_s( std_s = ekat::sqrt(ekat::max(0, ekat::square(cthl)*thl2 + ekat::square(cqt)*qw2 - 2*cthl*sqrtthl2*cqt*sqrtqw2*r_qwthl)); - const auto std_s_not_small = std_s > std::sqrt(std::numeric_limits::min()) * 100; + const auto std_s_not_small = std_s > std::sqrt(Kokkos::Experimental::norm_min_v) * 100; s = qw1-qs*((1 + beta*qw1)/(1 + beta*qs)); if (std_s_not_small.any()) { C.set(std_s_not_small, sp(0.5)*(1 + ekat::erf(s/(sqrt2*std_s)))); From 4e70071923f864138608c8bb2ea3eb9ff6a386ac Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 23 Aug 2024 07:03:19 -0700 Subject: [PATCH 367/477] fix typo in test mod --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index c293e611077e..86ee99886eb1 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -650,7 +650,7 @@ "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", - "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.small_kernels--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", ) }, From 5d5961c4a098d413a29280b1ab355a4ebd31679f Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 24 Aug 2024 14:59:28 -0700 Subject: [PATCH 368/477] add more testmods and better comments --- cime_config/tests.py | 2 ++ components/eamxx/src/scream_config.h.in | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 86ee99886eb1..5534a84e3c21 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -651,6 +651,8 @@ "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_p3--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_shoc--scream-output-preset-5", ) }, diff --git a/components/eamxx/src/scream_config.h.in b/components/eamxx/src/scream_config.h.in index a7c0f911f2e3..2bd8c72f013b 100644 --- a/components/eamxx/src/scream_config.h.in +++ b/components/eamxx/src/scream_config.h.in @@ -45,9 +45,11 @@ // Whether or not to run RRTMGP debug checks #cmakedefine SCREAM_RRTMGP_DEBUG -// Whether or not small kernels are on +// Whether or not small kernels are used in ALL targets that support them #cmakedefine SCREAM_SMALL_KERNELS +// Whether or not small kernels are used in P3 #cmakedefine SCREAM_P3_SMALL_KERNELS +// Whether or not small kernels are used in SHOC #cmakedefine SCREAM_SHOC_SMALL_KERNELS // The sha of the last commit From 4ee10629129d1fa52b0f525b9d079807713c3eae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:41:45 +0000 Subject: [PATCH 369/477] Bump tj-actions/changed-files from 44 to 45 Bumps [tj-actions/changed-files](https://github.com/tj-actions/changed-files) from 44 to 45. - [Release notes](https://github.com/tj-actions/changed-files/releases) - [Changelog](https://github.com/tj-actions/changed-files/blob/main/HISTORY.md) - [Commits](https://github.com/tj-actions/changed-files/compare/v44...v45) --- updated-dependencies: - dependency-name: tj-actions/changed-files dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-md-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 8ee8b7a7f13c..507c2c3745ea 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: tj-actions/changed-files@v44 + - uses: tj-actions/changed-files@v45 id: changed-files with: files: '**/*.md' From 9ed3ead2038dd963a242243a018f49895a52f60f Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 26 Aug 2024 14:13:17 -0400 Subject: [PATCH 370/477] clarify comment for cmake option --- components/eamxx/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index a60fc1e342fe..5715e5be1745 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -198,7 +198,7 @@ set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C inst set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) -set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for all components") +set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them") set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) From 8d324b21cd3bb90b1f8d28c5631e1a1b02788d42 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Mon, 26 Aug 2024 19:04:55 -0400 Subject: [PATCH 371/477] Add libfabric/1.15.2.0 module to frontier-scream-gpu. --- cime_config/machines/config_machines.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index d0e015349b3a..889fc03834e1 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1436,11 +1436,10 @@ craype-accel-amd-gfx90a rocm/5.4.0 libunwind/1.6.2 - - cce/15.0.1 - craype craype/2.7.20 - cray-mpich cray-mpich/8.1.26 + libfabric/1.15.2.0 + craype/2.7.20 + cray-mpich/8.1.26 cray-python/3.9.13.1 subversion/1.14.1 git/2.36.1 From 1848615f0968fa5fa35b2ab5317845e9ecc33326 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 27 Aug 2024 09:34:09 -0700 Subject: [PATCH 372/477] Pm-cpu must still use gcc-11.2 There are several libraries we use that were built with gcc-11. --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index d0e015349b3a..baa622fb513f 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -218,7 +218,7 @@ PrgEnv-gnu/8.5.0 - gcc/12.2.0 + gcc/11.2.0 cray-libsci/23.02.1.1 From 30efb49b035afdbf84cc06a2d8a61faaf505b9b6 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 27 Aug 2024 14:40:26 -0700 Subject: [PATCH 373/477] Undo gcc downgrade --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index baa622fb513f..d0e015349b3a 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -218,7 +218,7 @@ PrgEnv-gnu/8.5.0 - gcc/11.2.0 + gcc/12.2.0 cray-libsci/23.02.1.1 From ac671bd8d34c9e532978e4aded129955400ee307 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 28 Aug 2024 09:32:44 -0700 Subject: [PATCH 374/477] Do static libc++ linking --- cime_config/machines/cmake_macros/gnu_pm-cpu.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake b/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake index 226d07350a78..c2093f7f7fbb 100644 --- a/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake @@ -10,3 +10,6 @@ set(MPIFC "ftn") set(SCC "gcc") set(SCXX "g++") set(SFC "gfortran") + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -static-libstdc++") + From 004c1870aa962eab2287fc99b6831c000a4a0bab Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 28 Aug 2024 15:18:23 -0700 Subject: [PATCH 375/477] add several SHOC variables to be output for diagnostic purposes --- .../shoc/eamxx_shoc_process_interface.cpp | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 657f9487611c..f07912796002 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -91,6 +91,20 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("ustar", scalar2d, m/s, grid_name, ps); add_field("obklen", scalar2d, m, grid_name, ps); + // Diagnostic output - mid point grid + add_field("brunt", scalar3d_mid, pow(s,-1), grid_name, ps); + add_field("shoc_mix", scalar3d_mid, m, grid_name, ps); + add_field("isotropy", scalar3d_mid, s, grid_name, ps); + + // Diagnostic output - interface grid + add_field("wthl_sec", scalar3d_int, K*(m/s), grid_name, ps); + add_field("thl_sec", scalar3d_int, pow(K,2), grid_name, ps); + add_field("wqw_sec", scalar3d_int, (kg/kg)*(m/s), grid_name, ps); + add_field("qw_sec", scalar3d_int, pow(kg/kg,2), grid_name, ps); + add_field("uw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("vw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("w3", scalar3d_int, pow(m/s,3), grid_name, ps); + // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -495,6 +509,38 @@ void SHOCMacrophysics::run_impl (const double dt) default_policy, shoc_postprocess); Kokkos::fence(); + + // SHOC output diagnostics + const auto& shoc_mix = get_field_out("shoc_mix").get_view(); + Kokkos::deep_copy(shoc_mix,history_output.shoc_mix); + + const auto& brunt = get_field_out("brunt").get_view(); + Kokkos::deep_copy(brunt,history_output.brunt); + + const auto& w3 = get_field_out("w3").get_view(); + Kokkos::deep_copy(w3,history_output.w3); + + const auto& isotropy = get_field_out("isotropy").get_view(); + Kokkos::deep_copy(isotropy,history_output.isotropy); + + const auto& wthl_sec = get_field_out("wthl_sec").get_view(); + Kokkos::deep_copy(wthl_sec,history_output.wthl_sec); + + const auto& wqw_sec = get_field_out("wqw_sec").get_view(); + Kokkos::deep_copy(wqw_sec,history_output.wqw_sec); + + const auto& uw_sec = get_field_out("uw_sec").get_view(); + Kokkos::deep_copy(uw_sec,history_output.uw_sec); + + const auto& vw_sec = get_field_out("vw_sec").get_view(); + Kokkos::deep_copy(vw_sec,history_output.vw_sec); + + const auto& qw_sec = get_field_out("qw_sec").get_view(); + Kokkos::deep_copy(qw_sec,history_output.qw_sec); + + const auto& thl_sec = get_field_out("thl_sec").get_view(); + Kokkos::deep_copy(thl_sec,history_output.thl_sec); + } // ========================================================================================= void SHOCMacrophysics::finalize_impl() From 050d14990f5420c51fa55a03be332d49854d4a7e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 28 Aug 2024 16:12:23 -0700 Subject: [PATCH 376/477] Fixes namelist to enable fine grid simulations --- .../cime_config/namelist_defaults_scream.xml | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index a785fb322912..dadb6dbba35b 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,7 +269,7 @@ be lost if SCREAM_HACK_XML is not enabled. - + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc @@ -279,20 +279,19 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc - - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120np4_mono_20220502.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc From 78c4a4848a04a21fffc6100d5fb148f927775085 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Wed, 28 Aug 2024 22:01:41 -0500 Subject: [PATCH 377/477] remove some white spaces --- .../atmosphere_surface_coupling_importer.cpp | 4 +-- .../homme/interface/phys_grid_mod.F90 | 6 ++-- .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 36 +++++++++---------- .../mct_coupling/scream_cxx_f90_interface.cpp | 12 +++---- .../eamxx/src/mct_coupling/scream_f2c_mod.F90 | 2 +- .../src/share/atm_process/SCDataManager.hpp | 6 ++-- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 7686319ad3d8..ee3e21e7461b 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -80,7 +80,7 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr 0) then appname="HM_COARSE"//C_NULL_CHAR - ATM_ID1 = 120 ! + ATM_ID1 = 120 ! ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, MHID) if (ierr > 0 ) & call abortmp('Error: cannot register moab app') @@ -641,7 +641,7 @@ subroutine phys_grid_init (pgN) endif appname="HM_PGX"//C_NULL_CHAR ATM_ID1 = ATMID(1) ! this number should not conflict with other components IDs; how do we know? - ! + ! ! in this case, we reuse the main atm id, mhid will not be used for intersection anymore ! still, need to be careful ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, mhpgid) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index b5513afb4cca..9f976d6f0c27 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -19,7 +19,7 @@ module atm_comp_mct #ifdef HAVE_MOAB use seq_comm_mct , only: mphaid ! atm physics grid id in MOAB, on atm pes - use iso_c_binding + use iso_c_binding #ifdef MOABCOMP use seq_comm_mct, only: seq_comm_compare_mb_mct #endif @@ -55,7 +55,7 @@ module atm_comp_mct #ifdef HAVE_MOAB - ! to store all fields to be set in moab + ! to store all fields to be set in moab integer , private :: mblsize, totalmbls, nsend, totalmbls_r, nrecv real(r8) , allocatable, target, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes real(r8) , allocatable, target, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes @@ -63,7 +63,7 @@ module atm_comp_mct integer :: mpicom_atm_moab ! used just for mpi-reducing the difference between moab tags and mct avs integer :: rank2 #endif -#endif +#endif !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -128,7 +128,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) character(CL) :: calendar ! calendar string #ifdef HAVE_MOAB integer :: ent_type - character(CXX) :: tagname ! will store all seq_flds_a2x_fields + character(CXX) :: tagname ! will store all seq_flds_a2x_fields #ifdef MOABDEBUG character*100 outfile, wopts #endif @@ -210,7 +210,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call atm_domain_mct (lsize, gsMap_atm, dom_atm) #ifdef HAVE_MOAB - call moab_atm_phys_scream() + call moab_atm_phys_scream() #ifdef MOABCOMP mpicom_atm_moab = mpicom_atm ! just to store it to be used later in MOABCOMP rank2 = my_task ! again, just to use it for MOABCOMP output @@ -242,7 +242,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) a2x_am = 0. ! initialize everything with 0 nrecv = mct_avect_nRattr(x2a) - totalmbls_r = mblsize * nrecv ! size of the double array used to receive + totalmbls_r = mblsize * nrecv ! size of the double array used to receive allocate (x2a_am(mblsize, nrecv) ) ! these will be received by moab tags, then used to set cam in surf data x2a_am = 0. ! initialize everything with 0 #endif @@ -268,9 +268,9 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(trim(username),username_c) call string_f2c(trim(hostname),hostname_c) call scream_init_atm (caseid_c,hostname_c,username_c) -#ifdef HAVE_MOAB +#ifdef HAVE_MOAB ! data should be set now inside moab from import and export fields - ! do we import and export or just export at init stage ? + ! do we import and export or just export at init stage ? tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR ent_type = 0 ! vertices, point cloud ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, totalmbls , ent_type, a2x_am ) @@ -293,7 +293,7 @@ end subroutine atm_init_mct subroutine atm_run_mct(EClock, cdata, x2a, a2x) use iso_c_binding, only: c_double use scream_f2c_mod, only: scream_run -#ifdef HAVE_MOAB +#ifdef HAVE_MOAB use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields use shr_kind_mod , only: cxx=>shr_kind_cxx use iMOAB, only: iMOAB_SetDoubleTagStorage, iMOAB_GetDoubleTagStorage @@ -309,7 +309,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! character(CXX) :: mct_field, modelStr -#endif +#endif #endif ! !INPUT/OUTPUT PARAMETERS: @@ -325,7 +325,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) type(mct_gGrid) , pointer :: ggrid real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream -#ifdef HAVE_MOAB +#ifdef HAVE_MOAB integer :: ierr character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields #ifdef MOABDEBUG @@ -365,12 +365,12 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) tagname=trim(seq_flds_x2a_fields)//C_NULL_CHAR ent_type = 0 ! vertices, point cloud ierr = iMOAB_GetDoubleTagStorage ( mphaid, tagname, totalmbls_r , ent_type, x2a_am ) - + #endif ! Run scream call scream_run( dt_scream ) -#ifdef HAVE_MOAB +#ifdef HAVE_MOAB ! export data to moab data structures tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR ent_type = 0 ! vertices, point cloud @@ -517,9 +517,9 @@ subroutine moab_atm_phys_scream() #ifdef MOABDEBUG character*100 outfile, wopts #endif - character(CXX) :: tagname ! will store all seq_flds_a2x_fields + character(CXX) :: tagname ! will store all seq_flds_a2x_fields character*32 appname - integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) + integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) !------------------------------------------------------------------- ! ! Local Variables @@ -543,7 +543,7 @@ subroutine moab_atm_phys_scream() if (my_task == master_task) then print *, " moab_atm_phys_scream:: register MOAB app:", trim(appname), " mphaid=", mphaid endif - ! find global ids + ! find global ids num_local_cols = scream_get_num_local_cols() num_global_cols = scream_get_num_global_cols() @@ -644,7 +644,7 @@ subroutine moab_atm_phys_scream() call mpi_abort(mpicom_atm,ierr,mpi_ierr) endif #endif - ! define fields seq_flds_a2x_fields + ! define fields seq_flds_a2x_fields tagtype = 1 ! dense, double numco = 1 ! one value per vertex / entity tagname = trim(seq_flds_a2x_fields)//C_NULL_CHAR @@ -653,7 +653,7 @@ subroutine moab_atm_phys_scream() print *, "Error: fail to define seq_flds_a2x_fields " call mpi_abort(mpicom_atm,ierr,mpi_ierr) endif - ! make sure this is defined too; it could have the same fields, but in different order, or really different + ! make sure this is defined too; it could have the same fields, but in different order, or really different ! fields; need to make sure we have them tagname = trim(seq_flds_x2a_fields)//C_NULL_CHAR ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 52e0d692e0a5..4d70b3109f9e 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -165,18 +165,18 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, } void scream_setup_surface_coupling (const char*& import_field_names, int*& import_cpl_indices, - double*& x2a_ptr, + double*& x2a_ptr, #ifdef HAVE_MOAB double*& x2a_moab_ptr, -#endif +#endif int*& import_vector_components, double*& import_constant_multiple, bool*& do_import_during_init, const int& num_cpl_imports, const int& num_scream_imports, const int& import_field_size, char*& export_field_names, int*& export_cpl_indices, - double*& a2x_ptr, + double*& a2x_ptr, #ifdef HAVE_MOAB double*& a2x_moab_ptr, -#endif +#endif int*& export_vector_components, double*& export_constant_multiple, bool*& do_export_during_init, const int& num_cpl_exports, const int& num_scream_exports, const int& export_field_size) @@ -207,14 +207,14 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor num_cpl_imports, num_scream_imports, import_field_size, x2a_ptr, #ifdef HAVE_MOAB x2a_moab_ptr, -#endif +#endif names_in[0], import_cpl_indices, import_vector_components, import_constant_multiple, do_import_during_init); ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Export, num_cpl_exports, num_scream_exports, export_field_size, a2x_ptr, #ifdef HAVE_MOAB a2x_moab_ptr, -#endif +#endif names_out[0], export_cpl_indices, export_vector_components, export_constant_multiple, do_export_during_init); }); diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 5e09def98df0..74b45f8330b9 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -58,7 +58,7 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices a2x_ptr, & #ifdef HAVE_MOAB a2x_moab_ptr, & -#endif +#endif export_vector_components, & export_constant_multiple, do_export_during_init, & num_cpl_exports, num_scream_exports, export_field_size) bind(c) diff --git a/components/eamxx/src/share/atm_process/SCDataManager.hpp b/components/eamxx/src/share/atm_process/SCDataManager.hpp index 2dcfd3c5016c..76bc9e5bd843 100644 --- a/components/eamxx/src/share/atm_process/SCDataManager.hpp +++ b/components/eamxx/src/share/atm_process/SCDataManager.hpp @@ -22,10 +22,10 @@ struct SCDataManager { ~SCDataManager() = default; void setup_internals (const int num_cpl_fields, const int num_scream_fields, const int field_size, - Real* field_data_ptr, + Real* field_data_ptr, #ifdef HAVE_MOAB Real* field_data_moab_ptr, -#endif +#endif char* field_names, int* field_cpl_indices_ptr, int* field_vector_components_ptr, Real* field_constant_multiple_ptr, bool* transfer_during_init_ptr) @@ -117,7 +117,7 @@ struct SCDataManager { view_2d m_field_data; #ifdef HAVE_MOAB - view_2d m_field_data_moab; + view_2d m_field_data_moab; #endif name_t* m_field_names; view_1d m_field_cpl_indices; From d8d89e69a472ffb5e8f005bfdd20908e97c7a636 Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Thu, 29 Aug 2024 11:48:28 -0700 Subject: [PATCH 378/477] Update shell_commands to remove spa --- .../mam4xx/srf_online_emiss_constituent_fluxes/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands index 05b0b4954116..f47a1729ed74 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -11,7 +11,7 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/u #Update IC file and add the processes #------------------------------------------------------ $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,rrtmgp,mam4_srf_online_emiss" -b From 86b6965b6ca6aaea3f74c5e6ce07fe86e5dde736 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Aug 2024 13:44:57 -0600 Subject: [PATCH 379/477] Update rrtmgp branch --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 02c2e03df33e..8ff525eeed1d 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 02c2e03df33edca6101670168ca187cb94c0768e +Subproject commit 8ff525eeed1d87a2ca6f251c4d16b46222c5554d From 5afe73dcde08ea9c68fc3e8c4486bce49a1c5251 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Aug 2024 13:45:23 -0600 Subject: [PATCH 380/477] Set kokkos to be the default rrtmgp backend --- components/eamxx/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 398efa3bc2df..ac2737f2fb6e 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -210,8 +210,8 @@ endif() # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) -option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE) -option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE) if (SCREAM_RRTMGP_ENABLE_YAKL) add_definitions("-DRRTMGP_ENABLE_YAKL") endif() From 4b573528ff76ff23b3613792db02ee0dea9a4dbc Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Thu, 29 Aug 2024 12:50:13 -0700 Subject: [PATCH 381/477] Adds units for mass and number ofr constituent fluxes --- .../src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 96d26ca22f9e..3a1a1d39b63e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -86,6 +86,7 @@ void MAMConstituentFluxes::set_grids( add_field("phis", scalar2d, m2 / s2, grid_name); // Constituent fluxes at the surface (gasses and aerosols) + //[units: kg/m2/s (mass) or #/m2/s (number)] add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); From ead3cb7f5b1be6a86d01fcdaaf1ec5e230a22dd3 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 29 Aug 2024 13:55:07 -0700 Subject: [PATCH 382/477] add runtime guard around added diagnostics --- .../cime_config/namelist_defaults_scream.xml | 1 + .../shoc/eamxx_shoc_process_interface.cpp | 76 ++++++++++--------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index f09d10b0b320..ef11b40431ab 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -232,6 +232,7 @@ be lost if SCREAM_HACK_XML is not enabled. 7.0 0.1 0.1 + false diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index f07912796002..9a99d0f15992 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -91,19 +91,24 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("ustar", scalar2d, m/s, grid_name, ps); add_field("obklen", scalar2d, m, grid_name, ps); - // Diagnostic output - mid point grid - add_field("brunt", scalar3d_mid, pow(s,-1), grid_name, ps); - add_field("shoc_mix", scalar3d_mid, m, grid_name, ps); - add_field("isotropy", scalar3d_mid, s, grid_name, ps); - - // Diagnostic output - interface grid - add_field("wthl_sec", scalar3d_int, K*(m/s), grid_name, ps); - add_field("thl_sec", scalar3d_int, pow(K,2), grid_name, ps); - add_field("wqw_sec", scalar3d_int, (kg/kg)*(m/s), grid_name, ps); - add_field("qw_sec", scalar3d_int, pow(kg/kg,2), grid_name, ps); - add_field("uw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); - add_field("vw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); - add_field("w3", scalar3d_int, pow(m/s,3), grid_name, ps); + // Extra SHOC output diagnostics + if (m_params.get("extra_shoc_diags", false)) { + + // Diagnostic output - mid point grid + add_field("brunt", scalar3d_mid, pow(s,-1), grid_name, ps); + add_field("shoc_mix", scalar3d_mid, m, grid_name, ps); + add_field("isotropy", scalar3d_mid, s, grid_name, ps); + + // Diagnostic output - interface grid + add_field("wthl_sec", scalar3d_int, K*(m/s), grid_name, ps); + add_field("thl_sec", scalar3d_int, pow(K,2), grid_name, ps); + add_field("wqw_sec", scalar3d_int, (kg/kg)*(m/s), grid_name, ps); + add_field("qw_sec", scalar3d_int, pow(kg/kg,2), grid_name, ps); + add_field("uw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("vw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("w3", scalar3d_int, pow(m/s,3), grid_name, ps); + + } // Extra SHOC output diagnostics // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -510,37 +515,40 @@ void SHOCMacrophysics::run_impl (const double dt) shoc_postprocess); Kokkos::fence(); - // SHOC output diagnostics - const auto& shoc_mix = get_field_out("shoc_mix").get_view(); - Kokkos::deep_copy(shoc_mix,history_output.shoc_mix); + // Extra SHOC output diagnostics + if (m_params.get("extra_shoc_diags", false)) { + + const auto& shoc_mix = get_field_out("shoc_mix").get_view(); + Kokkos::deep_copy(shoc_mix,history_output.shoc_mix); - const auto& brunt = get_field_out("brunt").get_view(); - Kokkos::deep_copy(brunt,history_output.brunt); + const auto& brunt = get_field_out("brunt").get_view(); + Kokkos::deep_copy(brunt,history_output.brunt); - const auto& w3 = get_field_out("w3").get_view(); - Kokkos::deep_copy(w3,history_output.w3); + const auto& w3 = get_field_out("w3").get_view(); + Kokkos::deep_copy(w3,history_output.w3); - const auto& isotropy = get_field_out("isotropy").get_view(); - Kokkos::deep_copy(isotropy,history_output.isotropy); + const auto& isotropy = get_field_out("isotropy").get_view(); + Kokkos::deep_copy(isotropy,history_output.isotropy); - const auto& wthl_sec = get_field_out("wthl_sec").get_view(); - Kokkos::deep_copy(wthl_sec,history_output.wthl_sec); + const auto& wthl_sec = get_field_out("wthl_sec").get_view(); + Kokkos::deep_copy(wthl_sec,history_output.wthl_sec); - const auto& wqw_sec = get_field_out("wqw_sec").get_view(); - Kokkos::deep_copy(wqw_sec,history_output.wqw_sec); + const auto& wqw_sec = get_field_out("wqw_sec").get_view(); + Kokkos::deep_copy(wqw_sec,history_output.wqw_sec); - const auto& uw_sec = get_field_out("uw_sec").get_view(); - Kokkos::deep_copy(uw_sec,history_output.uw_sec); + const auto& uw_sec = get_field_out("uw_sec").get_view(); + Kokkos::deep_copy(uw_sec,history_output.uw_sec); - const auto& vw_sec = get_field_out("vw_sec").get_view(); - Kokkos::deep_copy(vw_sec,history_output.vw_sec); + const auto& vw_sec = get_field_out("vw_sec").get_view(); + Kokkos::deep_copy(vw_sec,history_output.vw_sec); - const auto& qw_sec = get_field_out("qw_sec").get_view(); - Kokkos::deep_copy(qw_sec,history_output.qw_sec); + const auto& qw_sec = get_field_out("qw_sec").get_view(); + Kokkos::deep_copy(qw_sec,history_output.qw_sec); - const auto& thl_sec = get_field_out("thl_sec").get_view(); - Kokkos::deep_copy(thl_sec,history_output.thl_sec); + const auto& thl_sec = get_field_out("thl_sec").get_view(); + Kokkos::deep_copy(thl_sec,history_output.thl_sec); + } // Extra SHOC output diagnostics } // ========================================================================================= void SHOCMacrophysics::finalize_impl() From 4c3e57509382eb889ac070036e5778918f304e6b Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Aug 2024 17:12:20 -0600 Subject: [PATCH 383/477] Allow option to force rrtmgp inputs to be layoutleft --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 260 ++++++++++-------- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 30 +- 2 files changed, 167 insertions(+), 123 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index e89e02fe301b..67536538b48e 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -23,6 +23,48 @@ using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; +namespace { + +struct ConvertToRrtmgpSubview +{ + int beg; + int ncol; + + template + View subview1d(const View& v) const { + return View(v, std::make_pair(beg, beg+ncol)); + } + + template + View subview2d_impl(const View& v, const int inner_dim) const { + return View(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); + } + +#ifdef RRTMGP_LAYOUT_LEFT + template + BufferView subview2d(const FieldView&, const BufferView& buffer_view, const int inner_dim) const { + return BufferView(buffer_view, std::make_pair(0, ncol), Kokkos::ALL); + } +#else + // Be sure to trim the excess + // items from the field manager views due to simd packs. If we don't trim, then + // check_range_k will fail due to looking at unused values. Once rrtmgp can handle + // packs, this won't be necessary. + template + FieldView subview2d(const FieldView& field_view, const BufferView&, const int inner_dim) const { + return subview2d_impl(field_view, inner_dim); + } +#endif + + template + View subview3d(const View& v) const { + // The range assumes these are buffer views, not fields + return View(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + } +}; + +} + RRTMGPRadiation:: RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) @@ -255,7 +297,12 @@ size_t RRTMGPRadiation::requested_buffer_size_in_bytes() const Buffer::num_3d_nlay_nswbands*m_col_chunk_size*(m_nlay)*m_nswbands + Buffer::num_3d_nlay_nlwbands*m_col_chunk_size*(m_nlay)*m_nlwbands + Buffer::num_3d_nlay_nswgpts*m_col_chunk_size*(m_nlay)*m_nswgpts + - Buffer::num_3d_nlay_nlwgpts*m_col_chunk_size*(m_nlay)*m_nlwgpts; + Buffer::num_3d_nlay_nlwgpts*m_col_chunk_size*(m_nlay)*m_nlwgpts * +#if defined(RRTMGP_ENABLE_YAKL) && defined(RRTMGP_ENABLE_KOKKOS) + 2; +#else + 1; +#endif return interface_request * sizeof(Real); } // RRTMGPRadiation::requested_buffer_size @@ -409,13 +456,7 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.cld_tau_lw_bnd.totElems(); #endif - // During the transition to kokkos, the buffer views/arrays will point to the same memory, - // so care is needed to avoid repeating calculations in such a way that answers change. - // Example: buff_view(x) += foo; - // Stuff like this cannot be done twice when both kokkos and yakl are enabled #ifdef RRTMGP_ENABLE_KOKKOS - mem = reinterpret_cast(buffer_manager.get_memory()); - // 1d arrays m_buffer.mu0_k = decltype(m_buffer.mu0_k)(mem, m_col_chunk_size); mem += m_buffer.mu0_k.size(); @@ -847,7 +888,11 @@ void RRTMGPRadiation::run_impl (const double dt) { this->log(LogLevel::debug, "[RRTMGP::run_impl] Col chunk beg,end: " + std::to_string(beg) + ", " + std::to_string(beg+ncol) + "\n"); - + // d_tint and d_dz are used in eamxx calls and therefore + // must be layout right + ulrreal2dk d_tint = ulrreal2dk(m_buffer.d_tint.data(), m_col_chunk_size, m_nlay+1); + ulrreal2dk d_dz = ulrreal2dk(m_buffer.d_dz.data(), m_col_chunk_size, m_nlay); + auto d_mu0 = m_buffer.cosine_zenith; #ifdef RRTMGP_ENABLE_YAKL // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. @@ -920,123 +965,68 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cld_tau_lw_gpt = subview_3d(m_buffer.cld_tau_lw_gpt); #endif #ifdef RRTMGP_ENABLE_KOKKOS - // If YAKL is on, we don't want aliased memory in both the yakl and kokos - // subviews, so make new views and deep_copy. Also, be sure to trim the excess - // items from the field manager views due to simd packs. If we don't trim, then - // check_range_k will fail due to looking at unused values. Once rrtmgp can handle - // packs, this won't be necessary. - auto subview_1dk = [&](const ureal1dk& v) -> ureal1dk { - ureal1dk subv(v, std::make_pair(beg, beg+ncol)); -#ifdef RRTMGP_ENABLE_YAKL - real1dk rv(v.label(), ncol); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_1dkc = [&](const cureal1dk& v) -> cureal1dk { - cureal1dk subv(v, std::make_pair(beg, beg+ncol)); -#ifdef RRTMGP_ENABLE_YAKL - creal1dk rv(v.label(), ncol); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_2dk = [&](const ureal2dk& v, const int inner_dim) -> ureal2dk { - ureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); -#ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_2dkc = [&](const cureal2dk& v, const int inner_dim) -> cureal2dk { - cureal2dk subv(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); -#ifdef RRTMGP_ENABLE_YAKL - creal2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_3dk = [&](const ureal3dk& v) -> ureal3dk { - ureal3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); // The range assumes these are buffer views, not fields -#ifdef RRTMGP_ENABLE_YAKL - real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; + ConvertToRrtmgpSubview conv = {beg, ncol}; // Note, ncol will not necessary be m_col_chunk_size because the number of cols // will not always be evenly divided by m_col_chunk_size. In most cases, the // extra space will not cause any problems, but it does sometimes. - auto p_lay_k = subview_2dkc(d_pmid, m_nlay); - auto t_lay_k = subview_2dkc(d_tmid, m_nlay); - auto p_lev_k = subview_2dkc(d_pint, m_nlay+1); - auto p_del_k = subview_2dkc(d_pdel, m_nlay); + auto p_lay_k = conv.subview2d(d_pmid, m_buffer.p_lay_k, m_nlay); + auto t_lay_k = conv.subview2d(d_tmid, m_buffer.t_lay_k, m_nlay); + auto p_lev_k = conv.subview2d(d_pint, m_buffer.p_lev_k, m_nlay+1); + auto z_del_k = conv.subview2d(d_dz, m_buffer.z_del_k, m_nlay); + auto p_del_k = conv.subview2d(d_pdel, m_buffer.p_del_k, m_nlay); + auto t_lev_k = conv.subview2d(d_tint, m_buffer.t_lev_k, m_nlay+1); auto sfc_alb_dir_k = m_buffer.sfc_alb_dir_k; auto sfc_alb_dif_k = m_buffer.sfc_alb_dif_k; - auto sfc_alb_dir_vis_k = subview_1dkc(d_sfc_alb_dir_vis); - auto sfc_alb_dir_nir_k = subview_1dkc(d_sfc_alb_dir_nir); - auto sfc_alb_dif_vis_k = subview_1dkc(d_sfc_alb_dif_vis); - auto sfc_alb_dif_nir_k = subview_1dkc(d_sfc_alb_dif_nir); - auto qc_k = subview_2dkc(d_qc, m_nlay); - auto nc_k = subview_2dkc(d_nc, m_nlay); - auto qi_k = subview_2dkc(d_qi, m_nlay); + auto sfc_alb_dir_vis_k = conv.subview1d(d_sfc_alb_dir_vis); + auto sfc_alb_dir_nir_k = conv.subview1d(d_sfc_alb_dir_nir); + auto sfc_alb_dif_vis_k = conv.subview1d(d_sfc_alb_dif_vis); + auto sfc_alb_dif_nir_k = conv.subview1d(d_sfc_alb_dif_nir); + auto qc_k = conv.subview2d(d_qc, m_buffer.qc_k, m_nlay); + auto nc_k = conv.subview2d(d_nc, m_buffer.nc_k, m_nlay); + auto qi_k = conv.subview2d(d_qi, m_buffer.qi_k, m_nlay); auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; - auto rel_k = subview_2dkc(d_rel, m_nlay); - auto rei_k = subview_2dkc(d_rei, m_nlay); - auto sw_flux_up_k = subview_2dk(d_sw_flux_up, m_nlay+1); - auto sw_flux_dn_k = subview_2dk(d_sw_flux_dn, m_nlay+1); - auto sw_flux_dn_dir_k = subview_2dk(d_sw_flux_dn_dir, m_nlay+1); - auto lw_flux_up_k = subview_2dk(d_lw_flux_up, m_nlay+1); - auto lw_flux_dn_k = subview_2dk(d_lw_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_up_k = subview_2dk(d_sw_clnclrsky_flux_up, m_nlay+1); - auto sw_clnclrsky_flux_dn_k = subview_2dk(d_sw_clnclrsky_flux_dn, m_nlay+1); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(d_sw_clnclrsky_flux_dn_dir, m_nlay+1); - auto sw_clrsky_flux_up_k = subview_2dk(d_sw_clrsky_flux_up, m_nlay+1); - auto sw_clrsky_flux_dn_k = subview_2dk(d_sw_clrsky_flux_dn, m_nlay+1); - auto sw_clrsky_flux_dn_dir_k = subview_2dk(d_sw_clrsky_flux_dn_dir, m_nlay+1); - auto sw_clnsky_flux_up_k = subview_2dk(d_sw_clnsky_flux_up, m_nlay+1); - auto sw_clnsky_flux_dn_k = subview_2dk(d_sw_clnsky_flux_dn, m_nlay+1); - auto sw_clnsky_flux_dn_dir_k = subview_2dk(d_sw_clnsky_flux_dn_dir, m_nlay+1); - auto lw_clnclrsky_flux_up_k = subview_2dk(d_lw_clnclrsky_flux_up, m_nlay+1); - auto lw_clnclrsky_flux_dn_k = subview_2dk(d_lw_clnclrsky_flux_dn, m_nlay+1); - auto lw_clrsky_flux_up_k = subview_2dk(d_lw_clrsky_flux_up, m_nlay+1); - auto lw_clrsky_flux_dn_k = subview_2dk(d_lw_clrsky_flux_dn, m_nlay+1); - auto lw_clnsky_flux_up_k = subview_2dk(d_lw_clnsky_flux_up, m_nlay+1); - auto lw_clnsky_flux_dn_k = subview_2dk(d_lw_clnsky_flux_dn, m_nlay+1); + auto rel_k = conv.subview2d(d_rel, m_buffer.eff_radius_qc_k, m_nlay); + auto rei_k = conv.subview2d(d_rei, m_buffer.eff_radius_qi_k, m_nlay); + auto sw_flux_up_k = conv.subview2d(d_sw_flux_up, m_buffer.sw_flux_up_k, m_nlay+1); + auto sw_flux_dn_k = conv.subview2d(d_sw_flux_dn, m_buffer.sw_flux_dn_k, m_nlay+1); + auto sw_flux_dn_dir_k = conv.subview2d(d_sw_flux_dn_dir, m_buffer.sw_flux_dn_dir_k, m_nlay+1); + auto lw_flux_up_k = conv.subview2d(d_lw_flux_up, m_buffer.lw_flux_up_k, m_nlay+1); + auto lw_flux_dn_k = conv.subview2d(d_lw_flux_dn, m_buffer.lw_flux_dn_k, m_nlay+1); + auto sw_clnclrsky_flux_up_k = conv.subview2d(d_sw_clnclrsky_flux_up, m_buffer.sw_clnclrsky_flux_up_k, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = conv.subview2d(d_sw_clnclrsky_flux_dn, m_buffer.sw_clnclrsky_flux_dn_k, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = conv.subview2d(d_sw_clnclrsky_flux_dn_dir, m_buffer.sw_clnclrsky_flux_dn_dir_k, m_nlay+1); + auto sw_clrsky_flux_up_k = conv.subview2d(d_sw_clrsky_flux_up, m_buffer.sw_clrsky_flux_up_k, m_nlay+1); + auto sw_clrsky_flux_dn_k = conv.subview2d(d_sw_clrsky_flux_dn, m_buffer.sw_clrsky_flux_dn_k, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = conv.subview2d(d_sw_clrsky_flux_dn_dir, m_buffer.sw_clrsky_flux_dn_dir_k, m_nlay+1); + auto sw_clnsky_flux_up_k = conv.subview2d(d_sw_clnsky_flux_up, m_buffer.sw_clnsky_flux_up_k, m_nlay+1); + auto sw_clnsky_flux_dn_k = conv.subview2d(d_sw_clnsky_flux_dn, m_buffer.sw_clnsky_flux_dn_k, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = conv.subview2d(d_sw_clnsky_flux_dn_dir, m_buffer.sw_clnsky_flux_dn_dir_k, m_nlay+1); + auto lw_clnclrsky_flux_up_k = conv.subview2d(d_lw_clnclrsky_flux_up, m_buffer.lw_clnclrsky_flux_up_k, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = conv.subview2d(d_lw_clnclrsky_flux_dn, m_buffer.lw_clnclrsky_flux_dn_k, m_nlay+1); + auto lw_clrsky_flux_up_k = conv.subview2d(d_lw_clrsky_flux_up, m_buffer.lw_clrsky_flux_up_k, m_nlay+1); + auto lw_clrsky_flux_dn_k = conv.subview2d(d_lw_clrsky_flux_dn, m_buffer.lw_clrsky_flux_dn_k, m_nlay+1); + auto lw_clnsky_flux_up_k = conv.subview2d(d_lw_clnsky_flux_up, m_buffer.lw_clnsky_flux_up_k, m_nlay+1); + auto lw_clnsky_flux_dn_k = conv.subview2d(d_lw_clnsky_flux_dn, m_buffer.lw_clnsky_flux_dn_k, m_nlay+1); auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; auto sw_bnd_flux_dif_k = m_buffer.sw_bnd_flux_dif_k; auto lw_bnd_flux_up_k = m_buffer.lw_bnd_flux_up_k; auto lw_bnd_flux_dn_k = m_buffer.lw_bnd_flux_dn_k; - auto sfc_flux_dir_vis_k = subview_1dk(d_sfc_flux_dir_vis); - auto sfc_flux_dir_nir_k = subview_1dk(d_sfc_flux_dir_nir); - auto sfc_flux_dif_vis_k = subview_1dk(d_sfc_flux_dif_vis); - auto sfc_flux_dif_nir_k = subview_1dk(d_sfc_flux_dif_nir); + auto sfc_flux_dir_vis_k = conv.subview1d(d_sfc_flux_dir_vis); + auto sfc_flux_dir_nir_k = conv.subview1d(d_sfc_flux_dir_nir); + auto sfc_flux_dif_vis_k = conv.subview1d(d_sfc_flux_dif_vis); + auto sfc_flux_dif_nir_k = conv.subview1d(d_sfc_flux_dif_nir); auto aero_tau_sw_k = m_buffer.aero_tau_sw_k; auto aero_ssa_sw_k = m_buffer.aero_ssa_sw_k; auto aero_g_sw_k = m_buffer.aero_g_sw_k; auto aero_tau_lw_k = m_buffer.aero_tau_lw_k; - auto cld_tau_sw_bnd_k = subview_3dk(m_buffer.cld_tau_sw_bnd_k); - auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); - auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); - auto cld_tau_lw_gpt_k = subview_3dk(m_buffer.cld_tau_lw_gpt_k); + auto cld_tau_sw_bnd_k = conv.subview3d(m_buffer.cld_tau_sw_bnd_k); + auto cld_tau_lw_bnd_k = conv.subview3d(m_buffer.cld_tau_lw_bnd_k); + auto cld_tau_sw_gpt_k = conv.subview3d(m_buffer.cld_tau_sw_gpt_k); + auto cld_tau_lw_gpt_k = conv.subview3d(m_buffer.cld_tau_lw_gpt_k); #endif - auto d_tint = m_buffer.d_tint; - auto d_dz = m_buffer.d_dz; - auto d_mu0 = m_buffer.cosine_zenith; // Set gas concs to "view" only the first ncol columns #ifdef RRTMGP_ENABLE_YAKL @@ -1045,7 +1035,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.ncol = ncol; - m_gas_concs_k.concs = subview_3dk(gas_concs_k); + m_gas_concs_k.concs = conv.subview3d(gas_concs_k); #endif // Copy data from the FieldManager to the YAKL arrays @@ -1156,6 +1146,26 @@ void RRTMGPRadiation::run_impl (const double dt) { } #endif #ifdef RRTMGP_ENABLE_KOKKOS +#ifdef RRTMGP_LAYOUT_LEFT + // Copy to layout left buffer views + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + p_lay_k(i,k) = d_pmid(icol,k); + t_lay_k(i,k) = d_tmid(icol,k); + z_del_k(i,k) = d_dz(i,k); + p_del_k(i,k) = d_pdel(icol,k); + qc_k(i,k) = d_qc(icol,k); + nc_k(i,k) = d_nc(icol,k); + qi_k(i,k) = d_qi(icol,k); + rel_k(i,k) = d_rel(icol,k); + rei_k(i,k) = d_rei(icol,k); + p_lev_k(i,k) = d_pint(icol,k); + t_lev_k(i,k) = d_tint(i,k); + }); + + p_lev_k(i,nlay) = d_pint(icol,nlay); + t_lev_k(i,nlay) = d_tint(i,nlay); +#endif + // Note that RRTMGP expects ordering (col,lay,bnd) but the FM keeps things in (col,bnd,lay) order if (do_aerosol_rad) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { @@ -1215,7 +1225,7 @@ void RRTMGPRadiation::run_impl (const double dt) { auto f = name=="o3" ? get_field_in(full_name) : get_field_out(full_name); auto d_vmr = f.get_view(); #ifdef RRTMGP_ENABLE_KOKKOS - auto tmp2d_k = subview_2dkc(d_vmr, m_nlay); + auto tmp2d_k = conv.subview2d_impl(d_vmr, m_nlay); #endif #ifdef RRTMGP_ENABLE_YAKL @@ -1328,11 +1338,8 @@ void RRTMGPRadiation::run_impl (const double dt) { iwp(i+1,k+1) *= 1e3; #endif #ifdef RRTMGP_ENABLE_KOKKOS -#ifndef RRTMGP_ENABLE_YAKL - // lwp and lwp_k point to the same memory lwp_k(i,k) *= 1e3; iwp_k(i,k) *= 1e3; -#endif #endif }); }); @@ -1385,7 +1392,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS interface_t::rrtmgp_main( ncol, m_nlay, - p_lay_k, t_lay_k, p_lev_k, d_tint, + p_lay_k, t_lay_k, p_lev_k, t_lev_k, m_gas_concs_k, sfc_alb_dir_k, sfc_alb_dif_k, d_mu0, lwp_k, iwp_k, rel_k, rei_k, cldfrac_tot_k, @@ -1592,7 +1599,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); interface_t::compute_aerocom_cloudtop( - ncol, nlay, t_lay_k, p_lay_k, p_del_k, d_dz, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, + ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k); @@ -1658,6 +1665,31 @@ void RRTMGPRadiation::run_impl (const double dt) { const int icol = i + beg; d_sfc_flux_sw_net(icol) = sw_flux_dn_k(i,kbot_k) - sw_flux_up_k(i,kbot_k); d_sfc_flux_lw_dn(icol) = lw_flux_dn_k(i,kbot_k); +#ifdef RRTMGP_LAYOUT_LEFT + // Copy from layout left buffer views back to layout right fields + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { + d_sw_flux_up(icol,k) = sw_flux_up_k(i,k); + d_sw_flux_dn(icol,k) = sw_flux_dn_k(i,k); + d_sw_flux_dn_dir(icol,k) = sw_flux_dn_dir_k(i,k); + d_lw_flux_up(icol,k) = lw_flux_up_k(i,k); + d_lw_flux_dn(icol,k) = lw_flux_dn_k(i,k); + d_sw_clnclrsky_flux_up(icol,k) = sw_clnclrsky_flux_up_k(i,k); + d_sw_clnclrsky_flux_dn(icol,k) = sw_clnclrsky_flux_dn_k(i,k); + d_sw_clnclrsky_flux_dn_dir(icol,k) = sw_clnclrsky_flux_dn_dir_k(i,k); + d_sw_clrsky_flux_up(icol,k) = sw_clrsky_flux_up_k(i,k); + d_sw_clrsky_flux_dn(icol,k) = sw_clrsky_flux_dn_k(i,k); + d_sw_clrsky_flux_dn_dir(icol,k) = sw_clrsky_flux_dn_dir_k(i,k); + d_sw_clnsky_flux_up(icol,k) = sw_clnsky_flux_up_k(i,k); + d_sw_clnsky_flux_dn(icol,k) = sw_clnsky_flux_dn_k(i,k); + d_sw_clnsky_flux_dn_dir(icol,k) = sw_clnsky_flux_dn_dir_k(i,k); + d_lw_clnclrsky_flux_up(icol,k) = lw_clnclrsky_flux_up_k(i,k); + d_lw_clnclrsky_flux_dn(icol,k) = lw_clnclrsky_flux_dn_k(i,k); + d_lw_clrsky_flux_up(icol,k) = lw_clrsky_flux_up_k(i,k); + d_lw_clrsky_flux_dn(icol,k) = lw_clrsky_flux_dn_k(i,k); + d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up_k(i,k); + d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn_k(i,k); + }); +#endif // Extract optical properties for COSP Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067_k); diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index e036e7ac08c0..329c7eeaba8d 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -14,15 +14,26 @@ namespace scream { * exactly ONE instance of this class in its list of subcomponents. */ +// rrtmgp is performance tuned for layout left views but will accept any +// view. We probably want to stick with layout left views for performance +// reasons even though this requires us to make copies of our fields (they +// are layout right). +#define RRTMGP_LAYOUT_LEFT + class RRTMGPRadiation : public AtmosphereProcess { public: - using KT = ekat::KokkosTypes; - using real1dk = typename KT::template view_1d; - using real2dk = typename KT::template view_2d; - using real3dk = typename KT::template view_3d; - using creal1dk = typename KT::template view_1d; - using creal2dk = typename KT::template view_2d; - using creal3dk = typename KT::template view_3d; + using KT = ekat::KokkosTypes; +#ifdef RRTMGP_LAYOUT_LEFT + using layout_t = Kokkos::LayoutLeft; +#else + using layout_t = typename ekat::KokkosTypes::Layout; +#endif + using real1dk = Kokkos::View; + using real2dk = Kokkos::View; + using real3dk = Kokkos::View; + using creal1dk = Kokkos::View; + using creal2dk = Kokkos::View; + using creal3dk = Kokkos::View; using ureal1dk = Unmanaged; using ureal2dk = Unmanaged; using ureal3dk = Unmanaged; @@ -32,7 +43,8 @@ class RRTMGPRadiation : public AtmosphereProcess { using ci_string = ekat::CaseInsensitiveString; - using layout_t = typename ekat::KokkosTypes::Layout; + using lrreal2dk = typename KT::template view_2d; + using ulrreal2dk = Unmanaged; #ifdef RRTMGP_ENABLE_KOKKOS using interface_t = rrtmgp::rrtmgp_interface; @@ -97,7 +109,7 @@ class RRTMGPRadiation : public AtmosphereProcess { // These are the gases that we keep track of int m_ngas; std::vector m_gas_names; - real1dk m_gas_mol_weights; + real1dk m_gas_mol_weights; #ifdef RRTMGP_ENABLE_YAKL GasConcs m_gas_concs; #endif From a42b2385ce3accc95cc664a24e11f15139479031 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 29 Aug 2024 18:12:29 -0600 Subject: [PATCH 384/477] EAMxx: use portable format for unsigned std::int64_t in printf --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 37cb251d7796..fe613259b163 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %" PRIx64 " (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + fprintf(stderr, "bfbhash> %14d %" PRIx64 " (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } From bdf22307e15e8cf3ca9f0cf58a9ceefae3d637d0 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 30 Aug 2024 22:28:18 -0500 Subject: [PATCH 385/477] change of parameters method create_moab_meshes is called from components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 in eamxx and from components/eam/src/dynamics/se/dyn_comp.F90 in eam in eamxx scenario, NPG (fv_nphys) is not visible in homme folder, so it has to be passed as a parameter to the routine This fix does not apply to eamxx, but when it will be merged back to master, it will need to be applied by eam, because homme implementation is changed --- components/eam/src/dynamics/se/dyn_comp.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/dynamics/se/dyn_comp.F90 b/components/eam/src/dynamics/se/dyn_comp.F90 index b420be4ee690..faff6ffd314b 100644 --- a/components/eam/src/dynamics/se/dyn_comp.F90 +++ b/components/eam/src/dynamics/se/dyn_comp.F90 @@ -269,7 +269,7 @@ subroutine dyn_init1(fh, NLFileName, dyn_in, dyn_out) end if #ifdef HAVE_MOAB - call create_moab_meshes(par, elem) + call create_moab_meshes(par, elem, fv_nphys) #endif ! Define the CAM grids (this has to be after dycore spinup). ! Physics-grid will be defined later by phys_grid_init From cf04c3890c3a99387401d4acdae3291514494a29 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sat, 31 Aug 2024 09:49:28 -0500 Subject: [PATCH 386/477] add more comments for moab functions --- .../src/dynamics/homme/interface/phys_grid_mod.F90 | 5 ++++- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 index 1ece3bafa724..31629c7c26ba 100644 --- a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 @@ -652,7 +652,10 @@ subroutine phys_grid_init (pgN) write(iulog,*) "register MOAB app:", trim(appname), " MHPGID=", mhpgid write(iulog,*) " " endif - +! instance distributed moab meshes from elem structures +! 1 ) spectral coarse mesh +! 2 ) GLL fine quad mesh (used mostly for visualization) +! 3 ) pgN FV type mesh, (most of the time pg2 mesh), used for coupling with other components; call create_moab_meshes(par, elem, pgN) endif #endif diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 9f976d6f0c27..ca7877a7bc3a 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -235,6 +235,11 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call scream_set_cpl_indices (x2a, a2x) #ifdef HAVE_MOAB +! for MOAB driver, need to setup surface coupling for MOAB arrays that will hold +! fields data for import and export +! setup is similar to setup for the attribute vectors real data arrays (x2a%rAttr +! and a2x%rAttr) +! the major difference is that moab arrays are transposed compared to AVs arrays mblsize = lsize nsend = mct_avect_nRattr(a2x) totalmbls = mblsize * nsend ! size of the double array @@ -497,6 +502,11 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) end subroutine atm_domain_mct #ifdef HAVE_MOAB +! as part of initialization, moab version of the MCT type grid needs to be instanced too +! it is what we call "point cloud mesh", it contains just the vertices corresponding to the +! grid for surface coupling +! its tags are corresponding to the fields from a2x and x2a attribute vectors, and its +! GLOBAL_ID tag corresponds to the global degrees of freedom in global atmosphere subroutine moab_atm_phys_scream() use iMOAB, only : iMOAB_RegisterApplication, iMOAB_CreateVertices, iMOAB_WriteMesh, & From 215d225dcfc04c4e9ebffa9b63c05d77b993e03a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 31 Aug 2024 23:40:20 -0700 Subject: [PATCH 387/477] Fixes path for files and change c-style strings to std::strings --- ...eamxx_mam_constituent_fluxes_interface.cpp | 30 ++++++++++--------- .../CMakeLists.txt | 18 +++++------ .../input.yaml | 18 +++++------ .../mam/emissions/CMakeLists.txt | 18 +++++------ .../single-process/mam/emissions/input.yaml | 18 +++++------ 5 files changed, 52 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 3a1a1d39b63e..543a445da989 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -100,7 +100,7 @@ void MAMConstituentFluxes::set_grids( // number (n) mixing ratios for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = + const std::string int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); @@ -108,31 +108,31 @@ void MAMConstituentFluxes::set_grids( // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - const char *cld_nmr_field_name = + const std::string cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const std::string int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); - if(strlen(int_mmr_field_name) > 0) { + if(not int_mmr_field_name.empty()) { add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - const char *cld_mmr_field_name = + const std::string cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); - if(strlen(cld_mmr_field_name) > 0) { + if(not cld_mmr_field_name.empty()) { add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } // end for loop num species } // end for loop for num modes for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // end for loop num gases @@ -217,36 +217,38 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const std::string int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const std::string cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const std::string int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if(not int_mmr_field_name.empty()) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = + const std::string cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if(not cld_mmr_field_name.empty()) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); } } } for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); } diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt index 8d2f2653387e..886936ca0846 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt @@ -22,15 +22,15 @@ GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 24c1d9f886e2..55c62d69aa24 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -14,15 +14,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index 3d64bc5b2b27..c8e690b929fc 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -27,15 +27,15 @@ GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 06be9ec8a883..e7af45178aa5 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,15 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From b1fbda11b93ed28d292291b36db2f6059fa5bd1b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 31 Aug 2024 23:56:03 -0700 Subject: [PATCH 388/477] Replaces preprocess interface code --- ...eamxx_mam_constituent_fluxes_interface.cpp | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 543a445da989..3efd57864fa3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -281,8 +281,26 @@ void MAMConstituentFluxes::run_impl(const double dt) { // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ // etc. - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); + + auto lambda = + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int i = team.league_rank(); // column index + compute_dry_mixing_ratios(team, wet_atm_, // in + dry_atm_, // out + i); // in + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, // in + dry_atm_, // out + i); // in + compute_updraft_velocities(team, wet_atm_, // in + dry_atm_, // out + i); // in + }; + Kokkos::parallel_for("mam_cfi_compute_updraft", scan_policy, lambda); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, constituent_fluxes_, From c6d07118118c5241ba8a923d513412b449bc9fb0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 1 Sep 2024 00:15:47 -0700 Subject: [PATCH 389/477] Removes preprocess function from hpp and a calls from cpp interface files --- ...eamxx_mam_constituent_fluxes_functions.hpp | 10 ++--- ...eamxx_mam_constituent_fluxes_interface.cpp | 32 ++++++--------- ...eamxx_mam_constituent_fluxes_interface.hpp | 41 ------------------- 3 files changed, 16 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index 12c9c172f17d..602fc9ebf894 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -19,10 +19,7 @@ void update_gas_aerosols_using_constituents( static constexpr int pcnst = mam4::aero_model::pcnst; // Declare local variables - auto wet_aero_loc = wet_aero; - auto dry_atm_loc = dry_atm; - const int surface_lev = nlev - 1; - auto constituent_fluxes_loc = constituent_fluxes; + const int surface_lev = nlev - 1; // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); @@ -34,8 +31,7 @@ void update_gas_aerosols_using_constituents( // Create a policy to loop over columns annd number of constituents // to update // FIXME: TODO:We don't need a team for "nconstituents", so we can make the - // kookos_for - // simple by using just ncols + // kookos_for simple by using just ncols const auto policy = ekat::ExeSpaceUtils:: get_default_team_policy(ncol, nconstituents); @@ -58,7 +54,7 @@ void update_gas_aerosols_using_constituents( atmosphere_for_column(dry_atm, // output icol); // input - // Form state%q like array + // Form state%q like array at surface level Real state_q_at_surf_lev[pcnst] = {}; mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, // input diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 3efd57864fa3..c168c1ef0e39 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -253,53 +253,47 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(gas_mmr_field_name).get_view(); } - //----------------------------------------------------------------- - // Setup preprocessing and post processing - //----------------------------------------------------------------- - preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); - } // end initialize_impl() // ================================================================ // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in // E3SM. In EAMxx, all constituents are considered "wet" (or have wet // mixing ratios), we are *not* doing any wet to dry conversions in the - // "preprocess" for this process. We are simply updating the MAM4xx - // tracers using the "constituent fluxes". + // for this process. We are simply updating the MAM4xx tracers using the + // "constituent fluxes". // We are converting wet atm to dry atm. Since we do not use or update // any of the water constituents (qc, qv, qi etc.), we should be okay // to do this conversion. We need to do this conversion as our function // are built following HAERO data structures. // ------------------------------------------------------------------- - // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ - // etc. - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); - + // Compute vertical layer heights and updraft velocity. We need these to fully + // populate dry_atm_, so that we can form a HAERO atmosphere object. HAERO + // atmosphere object is used to for state%q like array. auto lambda = KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int i = team.league_rank(); // column index + const int icol = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_, // in dry_atm_, // out - i); // in + icol); // in team.team_barrier(); // vertical heights has to be computed after computing dry mixing ratios // for atmosphere compute_vertical_layer_heights(team, // in dry_atm_, // out - i); // in + icol); // in compute_updraft_velocities(team, wet_atm_, // in dry_atm_, // out - i); // in + icol); // in }; + // policy + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + Kokkos::parallel_for("mam_cfi_compute_updraft", scan_policy, lambda); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index c68065f21788..7d54761a2975 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -69,47 +69,6 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; - // Atmosphere processes often have a pre-processing step that constructs - // required variables from the set of fields stored in the field manager. - // This functor implements this step, which is called during run_impl. - struct Preprocess { - Preprocess() = default; - // on host: initializes preprocess functor with necessary state data - void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere &wet_atm, - const mam_coupling::DryAtmosphere &dry_atm) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; - dry_atm_pre_ = dry_atm; - } - - KOKKOS_INLINE_FUNCTION - void operator()( - const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index - - compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - team.team_barrier(); - // vertical heights has to be computed after computing dry mixing ratios - // for atmosphere - compute_vertical_layer_heights(team, dry_atm_pre_, i); - compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - } // operator() - - // local variables for preprocess struct - // number of horizontal columns and vertical levels - int ncol_pre_, nlev_pre_; - - // local atmospheric and aerosol state data - mam_coupling::WetAtmosphere wet_atm_pre_; - mam_coupling::DryAtmosphere dry_atm_pre_; - }; // Preprocess - - private: - // preprocessing scratch pads - Preprocess preprocess_; - }; // MAMConstituentFluxes } // namespace scream From 2d24c11c798a4a8844f38b4e4a0c4bf4fd34760b Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Sun, 1 Sep 2024 23:48:46 -0600 Subject: [PATCH 390/477] EAMxx: Fix PRIx64 usage to include width 16. --- .../eamxx/src/share/atm_process/atmosphere_process_hash.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index fe613259b163..533090e4e7cb 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %" PRIx64 " (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16" PRIx64 " (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %" PRIx64 " (%s)\n", + fprintf(stderr, "bfbhash> %14d %16" PRIx64 " (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } From dacbb43e3c985ea97dde3d0f5210e8fc60c50e84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:00:56 +0000 Subject: [PATCH 391/477] Bump actions/setup-python from 5.1.1 to 5.2.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.1 to 5.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.1...v5.2.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 9ea25ae1864e..709fecc623ac 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index e6fb53ba29f5..7c9da3264d68 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 7f8f2a177515..8decb315c9e2 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -22,7 +22,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.11" - name: Run unit tests From ac1f0eda2e37b29f090013cdef3bb66534362860 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Fri, 30 Aug 2024 16:48:23 -0500 Subject: [PATCH 392/477] Homme: Fix planar-mode pgN get_latlon routines. Add is_planar block to gfr_f_get_(corner_)latlon. Add unit test for these routines. Clean up and add to some other unit tests. --- components/homme/src/share/gllfvremap_mod.F90 | 97 ++++++++++++++----- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e0e0fa6c4daa..e8013df217b1 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -2088,9 +2088,14 @@ subroutine gfr_f_get_latlon(ie, i, j, lat, lon) type (spherical_polar_t) :: p - p = change_coordinates(gfr%center_f(i,j,ie)) - lat = p%lat - lon = p%lon + if (gfr%is_planar) then + lon = gfr%center_f(i,j,ie)%x + lat = gfr%center_f(i,j,ie)%y + else + p = change_coordinates(gfr%center_f(i,j,ie)) + lat = p%lat + lon = p%lon + end if end subroutine gfr_f_get_latlon subroutine gfr_f_get_corner_latlon(ie, i, j, c, lat, lon) @@ -2103,9 +2108,14 @@ subroutine gfr_f_get_corner_latlon(ie, i, j, c, lat, lon) type (spherical_polar_t) :: p - p = change_coordinates(gfr%corners_f(c,i,j,ie)) - lat = p%lat - lon = p%lon + if (gfr%is_planar) then + lon = gfr%corners_f(c,i,j,ie)%x + lat = gfr%corners_f(c,i,j,ie)%y + else + p = change_coordinates(gfr%corners_f(c,i,j,ie)) + lat = p%lat + lon = p%lon + end if end subroutine gfr_f_get_corner_latlon subroutine gfr_f_get_cartesian3d(ie, i, j, p) @@ -2318,6 +2328,7 @@ subroutine set_ps_Q(elem, nets, nete, timeidx, qidx, nlev) ! Make up a test function for use in unit tests. use coordinate_systems_mod, only: cartesian3D_t, change_coordinates + use physical_constants, only: dd_pi, Lx, Ly type (element_t), intent(inout) :: elem(:) integer, intent(in) :: nets, nete, timeidx, qidx, nlev @@ -2329,7 +2340,13 @@ subroutine set_ps_Q(elem, nets, nete, timeidx, qidx, nlev) do ie = nets, nete do j = 1,np do i = 1,np - p = change_coordinates(elem(ie)%spherep(i,j)) + if (gfr%is_planar) then + p%x = elem(ie)%spherep(i,j)%lon*2*dd_pi/Lx + p%y = elem(ie)%spherep(i,j)%lat*2*dd_pi/Ly + p%z = 0 + else + p = change_coordinates(elem(ie)%spherep(i,j)) + end if elem(ie)%state%ps_v(i,j,timeidx) = & 1.0d3*(1 + 0.05*sin(2*p%x+0.5)*sin(p%y+1.5)*sin(3*p%z+2.5)) q = 0.5*(1 + sin(3*p%x)*sin(3*p%y)*sin(4*p%z)) @@ -2609,7 +2626,8 @@ function check(par, dom_mt, gfr, elem, verbose) result(nerr) logical, intent(in) :: verbose real(kind=real_kind) :: a, b, rd, x, y, f0(np*np), f1(np*np), g(np,np), & - wf(np*np), wg(np,np), qmin, qmax, qmin1, qmax1 + wf(np*np), wg(np,np), qmin, qmax, qmin1, qmax1, lat, lon, latext(2), & + lonext(2), tol integer :: nf, nf2, ie, i, j, k, iremap, info, ilimit, it real(kind=real_kind), allocatable :: Qdp_fv(:,:), ps_v_fv(:,:), & qmins(:,:,:), qmaxs(:,:,:) @@ -2669,6 +2687,31 @@ function check(par, dom_mt, gfr, elem, verbose) result(nerr) nerr = nerr + 1 write(iulog,*) 'gfr> Dinv', ie, rd end if + if (gfr%is_planar) then + lonext(1) = minval(elem(ie)%spherep(:,:)%lon) + lonext(2) = maxval(elem(ie)%spherep(:,:)%lon) + latext(1) = minval(elem(ie)%spherep(:,:)%lat) + latext(2) = maxval(elem(ie)%spherep(:,:)%lat) + tol = max(lonext(2) - lonext(1), latext(2) - latext(1))*1.e4*eps + do i = 1, nf + do j = 1, nf + call gfr_f_get_latlon(ie, i, j, lat, lon) + if ( lon < lonext(1) .or. lon > lonext(2) .or. & + lat < latext(1) .or. lat > latext(2)) then + write(iulog,*) 'gfr> planar latlon ctr', ie, latext, lonext, lat, lon + nerr = nerr + 1 + end if + do k = 1, 4 + call gfr_f_get_corner_latlon(ie, i, j, k, lat, lon) + if ( lon < lonext(1) - tol .or. lon > lonext(2) + tol .or. & + lat < latext(1) - tol .or. lat > latext(2) + tol) then + write(iulog,*) 'gfr> planar latlon crn', ie, latext, lonext, lat, lon + nerr = nerr + 1 + end if + end do + end do + end do + end if ! Check that FV -> GLL -> FV recovers the original FV values exactly ! (with no DSS and no limiter). @@ -2839,8 +2882,8 @@ function test_sphere2ref() result(nerr) use kinds, only: iulog type (cartesian3D_t) :: corners(4), sphere - real (real_kind) :: refin(2), refout(2), err - integer :: i, j, n, nerr + real (real_kind) :: refin(2), refout(2), err, tol + integer :: i, j, n, trial, nerr nerr = 0 @@ -2854,18 +2897,28 @@ function test_sphere2ref() result(nerr) do i = 1,n refin(1) = -1 + (1.0_real_kind/(n-1))*(i-1) do j = 1,n - refin(2) = -1 + (1.0_real_kind/(n-1))*(j-1) - call ref2sphere(corners, refin(1), refin(2), sphere) - call sphere2ref(corners, sphere, refout(1), refout(2)) - err = abs(refin(1) - refout(1)) + abs(refin(2) - refout(2)) - if (err > 15*eps .or. & - maxval(abs(refout)) > 1 + 5*eps .or. & - any(refout /= refout)) then - write(iulog,*) refin(1), refin(2) - write(iulog,*) refout(1), refout(2) - write(iulog,*) err - nerr = nerr + 1 - end if + do trial = 1, 2 + refin(2) = -1 + (1.0_real_kind/(n-1))*(j-1) + call ref2sphere(corners, refin(1), refin(2), sphere) + if (trial == 2) then + sphere%x = 0.9d0*sphere%x + sphere%y = 0.9d0*sphere%y + sphere%z = 0.9d0*sphere%z + end if + call sphere2ref(corners, sphere, refout(1), refout(2)) + err = abs(refin(1) - refout(1)) + abs(refin(2) - refout(2)) + tol = 15*eps + if (trial == 2) tol = 1e-6 + if (err > tol .or. & + maxval(abs(refout)) > 1 + 5*eps .or. & + any(refout /= refout)) then + write(iulog,*) 'gfr> test_sphere2ref trial', trial + write(iulog,*) refin(1), refin(2) + write(iulog,*) refout(1), refout(2) + write(iulog,*) err + nerr = nerr + 1 + end if + end do end do end do if (nerr /= 0) write(iulog,*) 'test_sphere2ref FAILED' From 6a2bebe06909386fdce735bd0d94b2c6750e8219 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 8 Aug 2024 13:09:08 -0400 Subject: [PATCH 393/477] Delete crusher from config_machines. --- cime_config/machines/config_machines.xml | 295 ----------------------- 1 file changed, 295 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 889fc03834e1..84cbaacd5506 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1094,301 +1094,6 @@ - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu - mpich - cli115 - /lustre/orion/cli115/world-shared/crusher - .* - /lustre/orion/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli115/world-shared/e3sm/baselines/crusher/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - 8 - 8 - 8 - TRUE - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - -c $ENV{OMP_NUM_THREADS} - $ENV{NTASKS_PER_GPU} - $ENV{GPU_BIND_ARGS} - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/15.0.1 - - - craype craype/2.7.20 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.4.0 - - - craype-accel-amd-gfx90a - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - gcc gcc/11.2.0 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - cray-python/3.9.13.1 - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0.25 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} - --gpu-bind=closest - romio_cb_read=disable - 0 - - - 10 - 3 - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --threads-per-core=1 - -c $ENV{OMP_NUM_THREADS} - -m *:block - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/14.0.0 - - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.1.0 - - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - - - cray-mpich/8.1.12 - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-libsci/21.08.1.2 - cray-hdf5-parallel/1.12.1.1 - cray-netcdf-hdf5parallel/4.8.1.1 - cray-parallel-netcdf/1.12.1.7 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - - 0 - - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 64 - 8 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --gpus-per-node=8 --gpu-bind=closest - -c $ENV{OMP_NUM_THREADS} - - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray - - craype-accel-amd-gfx90a - rocm/5.1.0 - - cce/14.0.3 - - - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - 0 - - 1 - romio_cb_read=disable - - - - 128M - spread - threads - - - Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* From bcc5b62f37e14c69a62cd7f57c11a9ec5d4ba5d8 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 8 Aug 2024 14:09:47 -0400 Subject: [PATCH 394/477] Deleted crusher files from cime_config/machines. --- .../Depends.crusher-gpu.crayclang.cmake | 43 --------- .../cmake_macros/amdclang_crusher.cmake | 12 --- .../cmake_macros/amdclanggpu_crusher.cmake | 26 ----- .../crayclang-scream_crusher-scream-cpu.cmake | 12 --- .../crayclang-scream_crusher-scream-gpu.cmake | 28 ------ .../crayclang-scream_crusher-scream.cmake | 12 --- .../cmake_macros/crayclang_crusher.cmake | 10 -- .../cmake_macros/crayclanggpu_crusher.cmake | 63 ------------- .../machines/cmake_macros/gnu_crusher.cmake | 15 --- .../cmake_macros/gnugpu_crusher.cmake | 25 ----- cime_config/machines/syslog.crusher | 94 ------------------- 11 files changed, 340 deletions(-) delete mode 100644 cime_config/machines/Depends.crusher-gpu.crayclang.cmake delete mode 100644 cime_config/machines/cmake_macros/amdclang_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/gnu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/gnugpu_crusher.cmake delete mode 100755 cime_config/machines/syslog.crusher diff --git a/cime_config/machines/Depends.crusher-gpu.crayclang.cmake b/cime_config/machines/Depends.crusher-gpu.crayclang.cmake deleted file mode 100644 index e41d959b52b4..000000000000 --- a/cime_config/machines/Depends.crusher-gpu.crayclang.cmake +++ /dev/null @@ -1,43 +0,0 @@ -set(CICE_F90 - ice_FY.F90 - ice_aerosol.F90 - ice_age.F90 - ice_atmo.F90 - ice_blocks.F90 - ice_calendar.F90 - ice_diagnostics.F90 - ice_distribution.F90 - ice_domain.F90 - ice_domain_size.F90 - ice_dyn_evp.F90 - ice_fileunits.F90 - ice_flux.F90 - ice_forcing.F90 - ice_grid.F90 - ice_history.F90 - ice_history_fields.F90 - ice_init.F90 - ice_itd.F90 - ice_kinds_mod.F90 - ice_lvl.F90 - ice_mechred.F90 - ice_meltpond.F90 - ice_ocean.F90 - ice_orbital.F90 - ice_probability.F90 - ice_probability_tools.F90 - ice_read_write.F90 - ice_restoring.F90 - ice_shortwave.F90 - ice_spacecurve.F90 - ice_state.F90 - ice_step_mod.F90 - ice_therm_itd.F90 - ice_therm_vertical.F90 - ice_transport_driver.F90 - ice_transport_remap.F90 - ice_work.F90) - -foreach(ITEM IN LISTS CICE_F90) - e3sm_add_flags("cice/src/source/${ITEM}" "-O0") -endforeach() diff --git a/cime_config/machines/cmake_macros/amdclang_crusher.cmake b/cime_config/machines/cmake_macros/amdclang_crusher.cmake deleted file mode 100644 index 2df5074d11f3..000000000000 --- a/cime_config/machines/cmake_macros/amdclang_crusher.cmake +++ /dev/null @@ -1,12 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") diff --git a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake deleted file mode 100644 index 6f3c0074798e..000000000000 --- a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake +++ /dev/null @@ -1,26 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") - -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") - -set(MPICXX "hipcc") -set(SCXX "hipcc") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake deleted file mode 100644 index f8176ccb3d0b..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() - -set(PIO_FILESYSTEM_HINTS "gpfs") - -string(APPEND CPPDEFS " -DCPRCRAY") - - diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake deleted file mode 100644 index 18564d4d301f..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake +++ /dev/null @@ -1,28 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() - -set(PIO_FILESYSTEM_HINTS "gpfs") - -set(MPICXX "hipcc") -set(SCXX "hipcc") - -string(APPEND CMAKE_CXX_FLAGS " -I${MPICH_DIR}/include") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.16/gtl/lib -lmpi_gtl_hsa") - -# For YAKL's -lroctx64 -lrocfft; the rocm module doesn't set this. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib") - -# 'NOT DEBUG': this resolves a crash in mct in docn init -# 'DEBUG' casee, too: resolves a build error in elm/src/main/elm_varctl.F90 due to several OpenACC syntax errors -#if (NOT DEBUG) - string(APPEND CMAKE_C_FLAGS " -O2 -hnoacc -hfp0 -hipa0") - string(APPEND CMAKE_Fortran_FLAGS " -O2 -hnoacc -hfp0 -hipa0") -#endif() - -string(APPEND CPPDEFS " -DCPRCRAY") - -#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "See SCREAM issue #2080.") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake deleted file mode 100644 index 2a0bfd6217c8..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -if (COMP_NAME STREQUAL elm) - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") - -set(PIO_FILESYSTEM_HINTS "gpfs") diff --git a/cime_config/machines/cmake_macros/crayclang_crusher.cmake b/cime_config/machines/cmake_macros/crayclang_crusher.cmake deleted file mode 100644 index 7a5fb412cbb9..000000000000 --- a/cime_config/machines/cmake_macros/crayclang_crusher.cmake +++ /dev/null @@ -1,10 +0,0 @@ -if (COMP_NAME STREQUAL elm) - # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -# Disable ipa and zero initialization are for other NaN isues: -# https://github.com/E3SM-Project/E3SM/pull/5208 -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") -# -em -ef generates modulename.mod (lowercase files) to support -# Scorpio installs -string(APPEND CMAKE_Fortran_FLAGS " -em -ef") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake deleted file mode 100644 index 653938b81da9..000000000000 --- a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake +++ /dev/null @@ -1,63 +0,0 @@ -set(MPICC "cc") -set(MPICXX "hipcc") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "hipcc") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() - -if (compile_threaded) - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") -string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY") -string(APPEND CMAKE_Fortran_FLAGS " -f free -em") -if (NOT compile_threaded) - # -M1077 flag used to suppress message about OpenMP directives - # that are ignored for non-threaded builds. (-h omp inactive) - # Details: `explain ftn-1077` - string(APPEND CMAKE_Fortran_FLAGS " -M1077") -endif() -set(HAS_F2008_CONTIGUOUS "TRUE") - -# -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: -# /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' -# AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") - -# Switch to O3 for better performance -# Using O2 to ensure passing tests -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") - -if (COMP_NAME STREQUAL elm) - # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -# -em -ef generates modulename.mod (lowercase files) to support -# Scorpio installs -# Disable ipa and zero initialization are for other NaN isues: -# https://github.com/E3SM-Project/E3SM/pull/5208 -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") - -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") - -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/cmake_macros/gnu_crusher.cmake b/cime_config/machines/cmake_macros/gnu_crusher.cmake deleted file mode 100644 index 9b242243989b..000000000000 --- a/cime_config/machines/cmake_macros/gnu_crusher.cmake +++ /dev/null @@ -1,15 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") - -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") diff --git a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake b/cime_config/machines/cmake_macros/gnugpu_crusher.cmake deleted file mode 100644 index bb213014b2f5..000000000000 --- a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake +++ /dev/null @@ -1,25 +0,0 @@ -set(MPICC "cc") -set(MPICXX "hipcc") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "hipcc") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") - -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--copy-dt-needed-entries -L/opt/cray/pe/gcc-libs -lgfortran -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") - -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/syslog.crusher b/cime_config/machines/syslog.crusher deleted file mode 100755 index 25ef50e1e955..000000000000 --- a/cime_config/machines/syslog.crusher +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/csh -f -# pm-gpu syslog script: -# mach_syslog - -set sample_interval = $1 -set jid = $2 -set lid = $3 -set run = $4 -set timing = $5 -set dir = $6 - -# Wait until job task-to-node mapping information is output before saving output file. -# Target length was determined empirically (maximum number of lines before job mapping -# information starts + number of nodes), and it may need to be adjusted in the future. -# (Note that calling script 'touch'es the e3sm log file before spawning this script, so that 'wc' does not fail.) -set nnodes = `scontrol show jobid $jid | grep -F NumNodes | sed 's/ *NumNodes=\([0-9]*\) .*/\1/' ` -@ target_lines = 150 + $nnodes -sleep 10 -set outlth = `wc \-l $run/e3sm.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -while ($outlth < $target_lines) - sleep 60 - set outlth = `wc \-l $run/e3sm.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -end - -set TimeLimit = `scontrol show jobid $jid | grep -F TimeLimit | sed 's/^ *RunTime=.*TimeLimit=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` -set limit_hours = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` -set limit_mins = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` -set limit_secs = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` -if ("X$limit_hours" == "X") set limit_hours = 0 -if ("X$limit_mins" == "X") set limit_mins = 0 -if ("X$limit_secs" == "X") set limit_secs = 0 -@ limit = 3600 * $limit_hours + 60 * $limit_mins + $limit_secs - -set RunTime = `scontrol show jobid $jid | grep -F RunTime | sed 's/^ *RunTime=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` -set runt_hours = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` -set runt_mins = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` -set runt_secs = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` -if ("X$runt_hours" == "X") set runt_hours = 0 -if ("X$runt_mins" == "X") set runt_mins = 0 -if ("X$runt_secs" == "X") set runt_secs = 0 -@ runt = 3600 * $runt_hours + 60 * $runt_mins + $runt_secs - -@ remaining = $limit - $runt -cat > $run/Walltime.Remaining < $dir/squeuef.$lid.$remaining - squeue -s | grep -v -F extern > $dir/squeues.$lid.$remaining - # squeue -t R -o "%.10i %R" > $dir/squeueR.$lid.$remaining -endif - -while ($remaining > 0) - echo "Wallclock time remaining: $remaining" >> $dir/atm.log.$lid.step - grep -Fa -e "nstep" -e "model date" $run/*atm.log.$lid | tail -n 4 >> $dir/atm.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/lnd.log.$lid.step - grep -Fa -e "timestep" -e "model date" $run/*lnd.log.$lid | tail -n 4 >> $dir/lnd.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/ocn.log.$lid.step - grep -Fa -e "timestep" -e "Step number" -e "model date" $run/*ocn.log.$lid | tail -n 4 >> $dir/ocn.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/ice.log.$lid.step - grep -Fa -e "timestep" -e "istep" -e "model date" $run/*ice.log.$lid | tail -n 4 >> $dir/ice.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/rof.log.$lid.step - grep -Fa "model date" $run/*rof.log.$lid | tail -n 4 >> $dir/rof.log.$lid.step - grep -Fa "model date" $run/*cpl.log.$lid > $dir/cpl.log.$lid.step-all - echo "Wallclock time remaining: $remaining" >> $dir/cpl.log.$lid.step - tail -n 4 $dir/cpl.log.$lid.step-all >> $dir/cpl.log.$lid.step - /bin/cp --preserve=timestamps -u $timing/* $dir - # sqs -w -a | grep "^[0-9]* *R *"> $dir/sqswr.$lid.$remaining - squeue -t R -o "%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l" > $dir/squeuef.$lid.$remaining - squeue -s | grep -v -F extern > $dir/squeues.$lid.$remaining - # squeue -t R -o "%.10i %R" > $dir/squeueR.$lid.$remaining - chmod a+r $dir/* - # sleep $sample_interval - set sleep_remaining = $sample_interval - while ($sleep_remaining > 120) - sleep 120 - @ sleep_remaining = $sleep_remaining - 120 - end - sleep $sleep_remaining - set RunTime = `scontrol show jobid $jid | grep -F RunTime | sed 's/^ *RunTime=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` - set runt_hours = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` - set runt_mins = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` - set runt_secs = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` - if ("X$runt_hours" == "X") set runt_hours = 0 - if ("X$runt_mins" == "X") set runt_mins = 0 - if ("X$runt_secs" == "X") set runt_secs = 0 - @ runt = 3600 * $runt_hours + 60 * $runt_mins + $runt_secs - @ remaining = $limit - $runt - cat > $run/Walltime.Remaining << EOF2 -$remaining $sample_interval -EOF2 - -end From ea640364780427fe3f6eac0e5bf6a8c1f58a0459 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 22 Aug 2024 14:32:05 -0400 Subject: [PATCH 395/477] Successful build and run of ne30pg2_ne30pg2 with PrgEnv-gnu and rocm/6.2.0 on Frontier. --- .../machines/cmake_macros/craygnu-hipcc.cmake | 37 +++++++++++++++++++ cime_config/machines/config_machines.xml | 30 +++++++-------- components/eamxx/CMakeLists.txt | 4 +- .../machine-files/frontier-scream-gpu.cmake | 2 +- 4 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 cime_config/machines/cmake_macros/craygnu-hipcc.cmake diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake new file mode 100644 index 000000000000..18f5300fdedc --- /dev/null +++ b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake @@ -0,0 +1,37 @@ +set(MPICC "cc") +set(MPICXX "hipcc") # Needs MPICH_CXX to use hipcc +set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90 +set(SCC "cc") +set(SCXX "hipcc") +set(SFC "ftn") + +string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND CMAKE_Fortran_FLAGS " -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fallow-argument-mismatch") + +string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow") +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace") + +string(APPEND CMAKE_C_FLAGS_RELEASE " -g -O2") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -g -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -g -O2") + +if (COMP_NAME STREQUAL csm_share) + string(APPEND CMAKE_C_FLAGS " -std=c99") +endif() +string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") +string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") + +string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") + +set(E3SM_LINK_WITH_FORTRAN "TRUE") +string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") + +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") + +set(USE_HIP "TRUE") +string(APPEND CMAKE_HIP_FLAGS "$ENV{CXXFLAGS} --offload-arch=gfx90a -munsafe-fp-atomics") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 84cbaacd5506..88fad58bf3e3 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1098,7 +1098,7 @@ Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* CNL - crayclang-scream + craygnu-hipcc mpich cli115 /lustre/orion/proj-shared/cli115 @@ -1135,23 +1135,21 @@ module module /usr/share/lmod/lmod/libexec/lmod python - + - cpe/22.12 + + PrgEnv-gnu + cpe/24.07 craype-accel-amd-gfx90a - rocm/5.4.0 - libunwind/1.6.2 - cce/15.0.1 - libfabric/1.15.2.0 - craype/2.7.20 - cray-mpich/8.1.26 - cray-python/3.9.13.1 - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 + rocm/6.2.0 + libunwind + cray-python + subversion + git + cmake + cray-hdf5-parallel + cray-netcdf-hdf5parallel + cray-parallel-netcdf darshan-runtime diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 398efa3bc2df..efaf77514d91 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -550,9 +550,9 @@ DisableMpiCxxBindings() if (SCREAM_DOUBLE_PRECISION) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") set(SCREAM_Fortran_FLAGS -real-size 64) - elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") + #elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") #find what is crayclang id in cmake - set(SCREAM_Fortran_FLAGS -s default32 -eZ) + #set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() diff --git a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake index 14d1d501160d..8704d848334b 100644 --- a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake +++ b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake @@ -6,4 +6,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/hip.cmake) set(SCREAM_MPIRUN_EXE "srun" CACHE STRING "") set(SCREAM_MACHINE "frontier-scream-gpu" CACHE STRING "") -set(CMAKE_CXX_FLAGS "--amdgpu-target=gfx90a -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-Wno-mismatched-tags --offload-arch=gfx90a -munsafe-fp-atomics -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) From dba1054f184ea62fbfe5eb07889e3ffd38a17c82 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Mon, 26 Aug 2024 10:53:31 -0400 Subject: [PATCH 396/477] Changed Frontier build to use OpenMP and Adios2. --- cime_config/machines/cmake_macros/craygnu-hipcc.cmake | 9 +++++++-- cime_config/machines/config_machines.xml | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake index 18f5300fdedc..8322c5d3a945 100644 --- a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake +++ b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake @@ -25,12 +25,17 @@ endif() string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - set(E3SM_LINK_WITH_FORTRAN "TRUE") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +if (compile_threaded) + string(APPEND CMAKE_C_FLAGS " -fopenmp") + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp=libgomp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") set(USE_HIP "TRUE") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 88fad58bf3e3..2448e2f797fc 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1176,8 +1176,8 @@ threads - - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi} From cc81b1f6565ef058eed6340a365e9eadc4d2a759 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 29 Aug 2024 20:21:38 -0400 Subject: [PATCH 397/477] Add SCREAM_SYSTEM_WORKAROUND_P3_PART2. Return support for crayclang-scream. Rename craygnuamdgpu. --- ...raygnu-hipcc.cmake => craygnuamdgpu.cmake} | 2 +- cime_config/machines/config_machines.xml | 29 +++++++++++++++++-- components/eamxx/CMakeLists.txt | 5 ++-- .../p3/disp/p3_main_impl_part2_disp.cpp | 7 ++++- 4 files changed, 35 insertions(+), 8 deletions(-) rename cime_config/machines/cmake_macros/{craygnu-hipcc.cmake => craygnuamdgpu.cmake} (97%) diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnuamdgpu.cmake similarity index 97% rename from cime_config/machines/cmake_macros/craygnu-hipcc.cmake rename to cime_config/machines/cmake_macros/craygnuamdgpu.cmake index 8322c5d3a945..6cb79c0146b1 100644 --- a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake +++ b/cime_config/machines/cmake_macros/craygnuamdgpu.cmake @@ -5,7 +5,7 @@ set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") -string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU") +string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU -DSCREAM_SYSTEM_WORKAROUND_P3_PART2") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 2448e2f797fc..e2d4a50cab55 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1098,7 +1098,7 @@ Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* CNL - craygnu-hipcc + craygnuamdgpu,crayclang-scream mpich cli115 /lustre/orion/proj-shared/cli115 @@ -1135,7 +1135,7 @@ module module /usr/share/lmod/lmod/libexec/lmod python - + PrgEnv-gnu @@ -1152,6 +1152,25 @@ cray-parallel-netcdf darshan-runtime + + + cpe/22.12 + craype-accel-amd-gfx90a + rocm/5.4.0 + libunwind/1.6.2 + cce/15.0.1 + libfabric/1.15.2.0 + craype/2.7.20 + cray-mpich/8.1.26 + cray-python/3.9.13.1 + subversion/1.14.1 + git/2.36.1 + cmake/3.21.3 + cray-hdf5-parallel/1.12.2.1 + cray-netcdf-hdf5parallel/4.9.0.1 + cray-parallel-netcdf/1.12.3.1 + darshan-runtime + $CIME_OUTPUT_ROOT/$CASE/run @@ -1176,9 +1195,13 @@ threads - + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + + diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index efaf77514d91..4c80aa55b4fd 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -550,9 +550,8 @@ DisableMpiCxxBindings() if (SCREAM_DOUBLE_PRECISION) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") set(SCREAM_Fortran_FLAGS -real-size 64) - #elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") - #find what is crayclang id in cmake - #set(SCREAM_Fortran_FLAGS -s default32 -eZ) + elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Cray") + set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 2b619d54bf33..7dddaf343e80 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -10,6 +10,9 @@ namespace p3 { * this file, #include p3_functions.hpp instead. */ +#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2 +#pragma clang optimize off +#endif template <> void Functions ::p3_main_part2_disp( @@ -130,7 +133,9 @@ ::p3_main_part2_disp( if (!hydrometeorsPresent(i)) return; }); } - +#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2 +#pragma clang optimize on +#endif } // namespace p3 } // namespace scream From 943ca3b61ec4f7298b0e90c5ecf30ab81ba80d32 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 30 Aug 2024 10:15:33 -0600 Subject: [PATCH 398/477] view alloc fixes for P3 small kernels --- .../src/physics/p3/disp/p3_main_impl_disp.cpp | 185 ++++++++++-------- .../physics/p3/eamxx_p3_process_interface.cpp | 124 +++++++++--- .../physics/p3/eamxx_p3_process_interface.hpp | 24 ++- .../eamxx/src/physics/p3/eamxx_p3_run.cpp | 6 +- .../src/physics/p3/impl/p3_main_impl.hpp | 30 +-- .../eamxx/src/physics/p3/p3_functions.hpp | 43 ++++ .../eamxx/src/physics/p3/p3_functions_f90.cpp | 37 +++- 7 files changed, 322 insertions(+), 127 deletions(-) diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index d20350b0788b..0f125b7ae960 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -32,16 +32,16 @@ ::p3_main_init_disp( const uview_2d& qv_supersat_i, const uview_2d& qtend_ignore, const uview_2d& ntend_ignore, const uview_2d& mu_c, const uview_2d& lamc, const uview_2d& rho_qi, const uview_2d& qv2qi_depos_tend, const uview_2d& precip_total_tend, const uview_2d& nevapr, const uview_2d& precip_liq_flux, const uview_2d& precip_ice_flux) -{ +{ using ExeSpace = typename KT::ExeSpace; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); Kokkos::parallel_for("p3_main_init", policy, KOKKOS_LAMBDA(const MemberType& team) { - - const Int i = team.league_rank(); - precip_liq_surf(i) = 0; - precip_ice_surf(i) = 0; + + const Int i = team.league_rank(); + precip_liq_surf(i) = 0; + precip_ice_surf(i) = 0; Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nk_pack), [&] (Int k) { @@ -107,6 +107,7 @@ ::p3_main_internal_disp( const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, + const P3Temporaries& temporaries, const WorkspaceManager& workspace_mgr, Int nj, Int nk, @@ -131,87 +132,99 @@ ::p3_main_internal_disp( view_1d nucleationPossible("nucleationPossible", nj); view_1d hydrometeorsPresent("hydrometeorsPresent", nj); - // - // Create temporary variables needed for p3 - // - view_2d - mu_r("mu_r", nj, nk_pack), // shape parameter of rain - T_atm("T_atm", nj, nk_pack), // temperature at the beginning of the microphysics step [K] - - // 2D size distribution and fallspeed parameters - lamr("lamr", nj, nk_pack), logn0r("logn0r", nj, nk_pack), nu("nu", nj, nk_pack), - cdist("cdist", nj, nk_pack), cdist1("cdist1", nj, nk_pack), cdistr("cdistr", nj, nk_pack), - - // Variables needed for in-cloud calculations - // Inverse cloud fractions (1/cld) - inv_cld_frac_i("inv_cld_frac_i", nj, nk_pack), inv_cld_frac_l("inv_cld_frac_l", nj, nk_pack), inv_cld_frac_r("inv_cld_frac_r", nj, nk_pack), - // In cloud mass-mixing ratios - qc_incld("qc_incld", nj, nk_pack), qr_incld("qr_incld", nj, nk_pack), qi_incld("qi_incld", nj, nk_pack), qm_incld("qm_incld", nj, nk_pack), - // In cloud number concentrations - nc_incld("nc_incld", nj, nk_pack), nr_incld("nr_incld", nj, nk_pack), ni_incld("ni_incld", nj, nk_pack), bm_incld("bm_incld", nj, nk_pack), - - // Other - inv_dz("inv_dz", nj, nk_pack), inv_rho("inv_rho", nj, nk_pack), ze_ice("ze_ice", nj, nk_pack), ze_rain("ze_rain", nj, nk_pack), - prec("prec", nj, nk_pack), rho("rho", nj, nk_pack), rhofacr("rhofacr", nj, nk_pack), rhofaci("rhofaci", nj, nk_pack), - acn("acn", nj, nk_pack), qv_sat_l("qv_sat", nj, nk_pack), qv_sat_i("qv_sat_i", nj, nk_pack), sup("sup", nj, nk_pack), - qv_supersat_i("qv_supersat", nj, nk_pack), tmparr2("tmparr2", nj, nk_pack), exner("exner", nj, nk_pack), - diag_equiv_reflectivity("diag_equiv_ref", nj, nk_pack), diag_vm_qi("diag_vm_qi", nj, nk_pack), diag_diam_qi("diag_diam_qi", nj, nk_pack), - pratot("pratot", nj, nk_pack), prctot("prctot", nj, nk_pack), - - // p3_tend_out, may not need these - qtend_ignore("qtend_ignore", nj, nk_pack), ntend_ignore("ntend_ignore", nj, nk_pack), - - // Variables still used in F90 but removed from C++ interface - mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), - qr_evap_tend("qr_evap_tend", nj, nk_pack), - - // cloud sedimentation - v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), - - // ice sedimentation - v_qit("v_qit", nj, nk_pack), v_nit("v_nit", nj, nk_pack), flux_nit("flux_nit", nj, nk_pack), flux_bir("flux_bir", nj, nk_pack), - flux_qir("flux_qir", nj, nk_pack), flux_qit("flux_qit", nj, nk_pack), - - // rain sedimentation - v_qr("v_qr", nj, nk_pack), v_nr("v_nr", nj, nk_pack); - // Get views of all inputs - auto pres = diagnostic_inputs.pres; - auto dz = diagnostic_inputs.dz; - auto nc_nuceat_tend = diagnostic_inputs.nc_nuceat_tend; - auto nccn_prescribed = diagnostic_inputs.nccn; - auto ni_activated = diagnostic_inputs.ni_activated; - auto inv_qc_relvar = diagnostic_inputs.inv_qc_relvar; - auto dpres = diagnostic_inputs.dpres; - auto inv_exner = diagnostic_inputs.inv_exner; - auto cld_frac_i = diagnostic_inputs.cld_frac_i; - auto cld_frac_l = diagnostic_inputs.cld_frac_l; - auto cld_frac_r = diagnostic_inputs.cld_frac_r; - auto col_location = infrastructure.col_location; - auto qc = prognostic_state.qc; - auto nc = prognostic_state.nc; - auto qr = prognostic_state.qr; - auto nr = prognostic_state.nr; - auto qi = prognostic_state.qi; - auto qm = prognostic_state.qm; - auto ni = prognostic_state.ni; - auto bm = prognostic_state.bm; - auto qv = prognostic_state.qv; - auto th = prognostic_state.th; - auto diag_eff_radius_qc = diagnostic_outputs.diag_eff_radius_qc; - auto diag_eff_radius_qi = diagnostic_outputs.diag_eff_radius_qi; - auto diag_eff_radius_qr = diagnostic_outputs.diag_eff_radius_qr; - auto qv2qi_depos_tend = diagnostic_outputs.qv2qi_depos_tend; - auto rho_qi = diagnostic_outputs.rho_qi; - auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; - auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; - auto precip_total_tend = diagnostic_outputs.precip_total_tend; - auto nevapr = diagnostic_outputs.nevapr; - auto qv_prev = diagnostic_inputs.qv_prev; - auto t_prev = diagnostic_inputs.t_prev; - auto liq_ice_exchange = history_only.liq_ice_exchange; - auto vap_liq_exchange = history_only.vap_liq_exchange; - auto vap_ice_exchange = history_only.vap_ice_exchange; + auto pres = diagnostic_inputs.pres; + auto dz = diagnostic_inputs.dz; + auto nc_nuceat_tend = diagnostic_inputs.nc_nuceat_tend; + auto nccn_prescribed = diagnostic_inputs.nccn; + auto ni_activated = diagnostic_inputs.ni_activated; + auto inv_qc_relvar = diagnostic_inputs.inv_qc_relvar; + auto dpres = diagnostic_inputs.dpres; + auto inv_exner = diagnostic_inputs.inv_exner; + auto cld_frac_i = diagnostic_inputs.cld_frac_i; + auto cld_frac_l = diagnostic_inputs.cld_frac_l; + auto cld_frac_r = diagnostic_inputs.cld_frac_r; + auto col_location = infrastructure.col_location; + auto qc = prognostic_state.qc; + auto nc = prognostic_state.nc; + auto qr = prognostic_state.qr; + auto nr = prognostic_state.nr; + auto qi = prognostic_state.qi; + auto qm = prognostic_state.qm; + auto ni = prognostic_state.ni; + auto bm = prognostic_state.bm; + auto qv = prognostic_state.qv; + auto th = prognostic_state.th; + auto diag_eff_radius_qc = diagnostic_outputs.diag_eff_radius_qc; + auto diag_eff_radius_qi = diagnostic_outputs.diag_eff_radius_qi; + auto diag_eff_radius_qr = diagnostic_outputs.diag_eff_radius_qr; + auto qv2qi_depos_tend = diagnostic_outputs.qv2qi_depos_tend; + auto rho_qi = diagnostic_outputs.rho_qi; + auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; + auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; + auto precip_total_tend = diagnostic_outputs.precip_total_tend; + auto nevapr = diagnostic_outputs.nevapr; + auto qv_prev = diagnostic_inputs.qv_prev; + auto t_prev = diagnostic_inputs.t_prev; + auto liq_ice_exchange = history_only.liq_ice_exchange; + auto vap_liq_exchange = history_only.vap_liq_exchange; + auto vap_ice_exchange = history_only.vap_ice_exchange; + auto mu_r = temporaries.mu_r; + auto T_atm = temporaries.T_atm; + auto lamr = temporaries.lamr; + auto logn0r = temporaries.logn0r; + auto nu = temporaries.nu; + auto cdist = temporaries.cdist; + auto cdist1 = temporaries.cdist1; + auto cdistr = temporaries.cdistr; + auto inv_cld_frac_i = temporaries.inv_cld_frac_i; + auto inv_cld_frac_l = temporaries.inv_cld_frac_l; + auto inv_cld_frac_r = temporaries.inv_cld_frac_r; + auto qc_incld = temporaries.qc_incld; + auto qr_incld = temporaries.qr_incld; + auto qi_incld = temporaries.qi_incld; + auto qm_incld = temporaries.qm_incld; + auto nc_incld = temporaries.nc_incld; + auto nr_incld = temporaries.nr_incld; + auto ni_incld = temporaries.ni_incld; + auto bm_incld = temporaries.bm_incld; + auto inv_dz = temporaries.inv_dz; + auto inv_rho = temporaries.inv_rho; + auto ze_ice = temporaries.ze_ice; + auto ze_rain = temporaries.ze_rain; + auto prec = temporaries.prec; + auto rho = temporaries.rho; + auto rhofacr = temporaries.rhofacr; + auto rhofaci = temporaries.rhofaci; + auto acn = temporaries.acn; + auto qv_sat_l = temporaries.qv_sat_l; + auto qv_sat_i = temporaries.qv_sat_i; + auto sup = temporaries.sup; + auto qv_supersat_i = temporaries.qv_supersat_i; + auto tmparr2 = temporaries.tmparr2; + auto exner = temporaries.exner; + auto diag_equiv_reflectivity = temporaries.diag_equiv_reflectivity; + auto diag_vm_qi = temporaries.diag_vm_qi; + auto diag_diam_qi = temporaries.diag_diam_qi; + auto pratot = temporaries.pratot; + auto prctot = temporaries.prctot; + auto qtend_ignore = temporaries.qtend_ignore; + auto ntend_ignore = temporaries.ntend_ignore; + auto mu_c = temporaries.mu_c; + auto lamc = temporaries.lamc; + auto qr_evap_tend = temporaries.qr_evap_tend; + auto v_qc = temporaries.v_qc; + auto v_nc = temporaries.v_nc; + auto flux_qx = temporaries.flux_qx; + auto flux_nx = temporaries.flux_nx; + auto v_qit = temporaries.v_qit; + auto v_nit = temporaries.v_nit; + auto flux_nit = temporaries.flux_nit; + auto flux_bir = temporaries.flux_bir; + auto flux_qir = temporaries.flux_qir; + auto flux_qit = temporaries.flux_qit; + auto v_qr = temporaries.v_qr; + auto v_nr = temporaries.v_nr; // we do not want to measure init stuff auto start = std::chrono::steady_clock::now(); @@ -242,7 +255,7 @@ ::p3_main_internal_disp( p3_main_part2_disp( nj, nk, runtime_options.max_total_ni, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, inv_dt, - lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, + lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, lookup_tables.revap_table_vals, pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index acaecd7eef70..c414d7093f62 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -152,10 +152,14 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) Real* mem = reinterpret_cast(buffer_manager.get_memory()); // 1d scalar views - m_buffer.precip_liq_surf_flux = decltype(m_buffer.precip_liq_surf_flux)(mem, m_num_cols); - mem += m_buffer.precip_liq_surf_flux.size(); - m_buffer.precip_ice_surf_flux = decltype(m_buffer.precip_ice_surf_flux)(mem, m_num_cols); - mem += m_buffer.precip_ice_surf_flux.size(); + using scalar_1d_view_t = decltype(m_buffer.precip_liq_surf_flux); + scalar_1d_view_t* _1d_scalar_view_ptrs[Buffer::num_1d_scalar] = { + &m_buffer.precip_liq_surf_flux, &m_buffer.precip_ice_surf_flux + }; + for (int i=0; isize(); + } // 2d scalar views m_buffer.col_location = decltype(m_buffer.col_location)(mem, m_num_cols, 3); @@ -167,26 +171,38 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) const Int nk_pack = ekat::npack(m_num_levs); const Int nk_pack_p1 = ekat::npack(m_num_levs+1); - m_buffer.inv_exner = decltype(m_buffer.inv_exner)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.inv_exner.size(); - m_buffer.th_atm = decltype(m_buffer.th_atm)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.th_atm.size(); - m_buffer.cld_frac_l = decltype(m_buffer.cld_frac_l)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.cld_frac_l.size(); - m_buffer.cld_frac_i = decltype(m_buffer.cld_frac_i)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.cld_frac_i.size(); - m_buffer.dz = decltype(m_buffer.dz)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.dz.size(); - m_buffer.qv2qi_depos_tend = decltype(m_buffer.qv2qi_depos_tend)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.qv2qi_depos_tend.size(); - m_buffer.rho_qi = decltype(m_buffer.rho_qi)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.rho_qi.size(); - m_buffer.precip_liq_flux = decltype(m_buffer.precip_liq_flux)(s_mem, m_num_cols, nk_pack_p1); - s_mem += m_buffer.precip_liq_flux.size(); - m_buffer.precip_ice_flux = decltype(m_buffer.precip_ice_flux)(s_mem, m_num_cols, nk_pack_p1); - s_mem += m_buffer.precip_ice_flux.size(); - m_buffer.unused = decltype(m_buffer.unused)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.unused.size(); + using spack_2d_view_t = decltype(m_buffer.inv_exner); + spack_2d_view_t* _2d_spack_mid_view_ptrs[Buffer::num_2d_vector] = { + &m_buffer.inv_exner, &m_buffer.th_atm, &m_buffer.cld_frac_l, &m_buffer.cld_frac_i, + &m_buffer.dz, &m_buffer.qv2qi_depos_tend, &m_buffer.rho_qi, &m_buffer.unused +#ifdef SCREAM_P3_SMALL_KERNELS + , &m_buffer.mu_r, &m_buffer.T_atm, &m_buffer.lamr, &m_buffer.logn0r, &m_buffer.nu, + &m_buffer.cdist, &m_buffer.cdist1, &m_buffer.cdistr, &m_buffer.inv_cld_frac_i, + &m_buffer.inv_cld_frac_l, &m_buffer.inv_cld_frac_r, &m_buffer.qc_incld, &m_buffer.qr_incld, + &m_buffer.qi_incld, &m_buffer.qm_incld, &m_buffer.nc_incld, &m_buffer.nr_incld, + &m_buffer.ni_incld, &m_buffer.bm_incld, &m_buffer.inv_dz, &m_buffer.inv_rho, &m_buffer.ze_ice, + &m_buffer.ze_rain, &m_buffer.prec, &m_buffer.rho, &m_buffer.rhofacr, &m_buffer.rhofaci, + &m_buffer.acn, &m_buffer.qv_sat_l, &m_buffer.qv_sat_i, &m_buffer.sup, &m_buffer.qv_supersat_i, + &m_buffer.tmparr2, &m_buffer.exner, &m_buffer.diag_equiv_reflectivity, &m_buffer.diag_vm_qi, + &m_buffer.diag_diam_qi, &m_buffer.pratot, &m_buffer.prctot, &m_buffer.qtend_ignore, + &m_buffer.ntend_ignore, &m_buffer.mu_c, &m_buffer.lamc, &m_buffer.qr_evap_tend, &m_buffer.v_qc, + &m_buffer.v_nc, &m_buffer.flux_qx, &m_buffer.flux_nx, &m_buffer.v_qit, &m_buffer.v_nit, + &m_buffer.flux_nit, &m_buffer.flux_bir, &m_buffer.flux_qir, &m_buffer.flux_qit, &m_buffer.v_qr, + &m_buffer.v_nr +#endif + }; + for (int i=0; isize(); + } + + spack_2d_view_t* _2d_spack_int_view_ptrs[Buffer::num_2dp1_vector] = { + &m_buffer.precip_liq_flux, &m_buffer.precip_ice_flux + }; + for (int i=0; isize(); + } // WSM data m_buffer.wsm_data = s_mem; @@ -324,6 +340,66 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) history_only.liq_ice_exchange = get_field_out("micro_liq_ice_exchange").get_view(); history_only.vap_liq_exchange = get_field_out("micro_vap_liq_exchange").get_view(); history_only.vap_ice_exchange = get_field_out("micro_vap_ice_exchange").get_view(); +#ifdef SCREAM_P3_SMALL_KERNELS + // Temporaries + temporaries.mu_r = m_buffer.mu_r; + temporaries.T_atm = m_buffer.T_atm; + temporaries.lamr = m_buffer.lamr; + temporaries.logn0r = m_buffer.logn0r; + temporaries.nu = m_buffer.nu; + temporaries.cdist = m_buffer.cdist; + temporaries.cdist1 = m_buffer.cdist1; + temporaries.cdistr = m_buffer.cdistr; + temporaries.inv_cld_frac_i = m_buffer.inv_cld_frac_i; + temporaries.inv_cld_frac_l = m_buffer.inv_cld_frac_l; + temporaries.inv_cld_frac_r = m_buffer.inv_cld_frac_r; + temporaries.qc_incld = m_buffer.qc_incld; + temporaries.qr_incld = m_buffer.qr_incld; + temporaries.qi_incld = m_buffer.qi_incld; + temporaries.qm_incld = m_buffer.qm_incld; + temporaries.nc_incld = m_buffer.nc_incld; + temporaries.nr_incld = m_buffer.nr_incld; + temporaries.ni_incld = m_buffer.ni_incld; + temporaries.bm_incld = m_buffer.bm_incld; + temporaries.inv_dz = m_buffer.inv_dz; + temporaries.inv_rho = m_buffer.inv_rho; + temporaries.ze_ice = m_buffer.ze_ice; + temporaries.ze_rain = m_buffer.ze_rain; + temporaries.prec = m_buffer.prec; + temporaries.rho = m_buffer.rho; + temporaries.rhofacr = m_buffer.rhofacr; + temporaries.rhofaci = m_buffer.rhofaci; + temporaries.acn = m_buffer.acn; + temporaries.qv_sat_l = m_buffer.qv_sat_l; + temporaries.qv_sat_i = m_buffer.qv_sat_i; + temporaries.sup = m_buffer.sup; + temporaries.qv_supersat_i = m_buffer.qv_supersat_i; + temporaries.tmparr2 = m_buffer.tmparr2; + temporaries.exner = m_buffer.exner; + temporaries.diag_equiv_reflectivity = m_buffer.diag_equiv_reflectivity; + temporaries.diag_vm_qi = m_buffer.diag_vm_qi; + temporaries.diag_diam_qi = m_buffer.diag_diam_qi; + temporaries.pratot = m_buffer.pratot; + temporaries.prctot = m_buffer.prctot; + temporaries.qtend_ignore = m_buffer.qtend_ignore; + temporaries.ntend_ignore = m_buffer.ntend_ignore; + temporaries.mu_c = m_buffer.mu_c; + temporaries.lamc = m_buffer.lamc; + temporaries.qr_evap_tend = m_buffer.qr_evap_tend; + temporaries.v_qc = m_buffer.v_qc; + temporaries.v_nc = m_buffer.v_nc; + temporaries.flux_qx = m_buffer.flux_qx; + temporaries.flux_nx = m_buffer.flux_nx; + temporaries.v_qit = m_buffer.v_qit; + temporaries.v_nit = m_buffer.v_nit; + temporaries.flux_nit = m_buffer.flux_nit; + temporaries.flux_bir = m_buffer.flux_bir; + temporaries.flux_qir = m_buffer.flux_qir; + temporaries.flux_qit = m_buffer.flux_qit; + temporaries.v_qr = m_buffer.v_qr; + temporaries.v_nr = m_buffer.v_nr; +#endif + // -- Set values for the post-amble structure p3_postproc.set_variables(m_num_cols,nk_pack, prog_state.th,pmid,pmid_dry,T_atm,t_prev, diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp index 47d543ded022..32e3b76d1a0d 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp @@ -158,7 +158,7 @@ class P3Microphysics : public AtmosphereProcess // Assigning local variables void set_variables(const int ncol, const int npack, const view_2d_const& pmid_, const view_2d_const& pmid_dry_, - const view_2d_const& pseudo_density_, + const view_2d_const& pseudo_density_, const view_2d_const& pseudo_density_dry_, const view_2d& T_atm_, const view_2d_const& cld_frac_t_, const view_2d& qv_, const view_2d& qc_, const view_2d& nc_, const view_2d& qr_, const view_2d& nr_, const view_2d& qi_, @@ -361,7 +361,11 @@ class P3Microphysics : public AtmosphereProcess // 1d view scalar, size (ncol) static constexpr int num_1d_scalar = 2; //no 2d vars now, but keeping 1d struct for future expansion // 2d view packed, size (ncol, nlev_packs) +#ifdef SCREAM_P3_SMALL_KERNELS + static constexpr int num_2d_vector = 64; +#else static constexpr int num_2d_vector = 8; +#endif static constexpr int num_2dp1_vector = 2; uview_1d precip_liq_surf_flux; @@ -377,6 +381,21 @@ class P3Microphysics : public AtmosphereProcess uview_2d precip_ice_flux; //nlev+1 uview_2d unused; +#ifdef SCREAM_P3_SMALL_KERNELS + uview_2d + mu_r, T_atm, lamr, logn0r, nu, cdist, cdist1, cdistr, + inv_cld_frac_i, inv_cld_frac_l, inv_cld_frac_r, + qc_incld, qr_incld, qi_incld, qm_incld, + nc_incld, nr_incld, ni_incld, bm_incld, + inv_dz, inv_rho, ze_ice, ze_rain, prec, rho, rhofacr, + rhofaci, acn, qv_sat_l, qv_sat_i, sup, qv_supersat_i, + tmparr2, exner, diag_equiv_reflectivity, diag_vm_qi, + diag_diam_qi, pratot, prctot, qtend_ignore, ntend_ignore, + mu_c, lamc, qr_evap_tend, v_qc, v_nc, flux_qx, flux_nx, + v_qit, v_nit, flux_nit, flux_bir, flux_qir, flux_qit, + v_qr, v_nr; +#endif + suview_2d col_location; Spack* wsm_data; @@ -410,6 +429,9 @@ class P3Microphysics : public AtmosphereProcess P3F::P3DiagnosticOutputs diag_outputs; P3F::P3HistoryOnly history_only; P3F::P3LookupTables lookup_tables; +#ifdef SCREAM_P3_SMALL_KERNELS + P3F::P3Temporaries temporaries; +#endif P3F::P3Infrastructure infrastructure; P3F::P3Runtime runtime_options; p3_preamble p3_preproc; diff --git a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp index 9ccfdaf20220..cdb156d71f62 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp @@ -29,7 +29,11 @@ void P3Microphysics::run_impl (const double dt) get_field_out("micro_vap_ice_exchange").deep_copy(0.0); P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, m_num_cols, m_num_levs, m_p3constants); + history_only, lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + temporaries, +#endif + workspace_mgr, m_num_cols, m_num_levs, m_p3constants); // Conduct the post-processing of the p3_main output. Kokkos::parallel_for( diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index c98a37d0557a..270042c99986 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -162,10 +162,10 @@ ::p3_main_internal( &inv_dz, &inv_rho, &ze_ice, &ze_rain, &prec, &rho, &rhofacr, &rhofaci, &acn, &qv_sat_l, &qv_sat_i, &sup, &qv_supersat_i, &tmparr1, &exner, &diag_equiv_reflectivity, &diag_vm_qi, &diag_diam_qi, - &pratot, &prctot, &qtend_ignore, &ntend_ignore, + &pratot, &prctot, &qtend_ignore, &ntend_ignore, &mu_c, &lamc, &qr_evap_tend }); - + // Get single-column subviews of all inputs, shouldn't need any i-indexing // after this. const auto opres = ekat::subview(diagnostic_inputs.pres, i); @@ -344,22 +344,15 @@ ::p3_main( const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + const P3Temporaries& temporaries, +#endif const WorkspaceManager& workspace_mgr, Int nj, Int nk, const physics::P3_Constants & p3constants) { -#ifndef SCREAM_P3_SMALL_KERNELS - return p3_main_internal(runtime_options, - prognostic_state, - diagnostic_inputs, - diagnostic_outputs, - infrastructure, - history_only, - lookup_tables, - workspace_mgr, - nj, nk, p3constants); -#else +#ifdef SCREAM_P3_SMALL_KERNELS return p3_main_internal_disp(runtime_options, prognostic_state, diagnostic_inputs, @@ -367,8 +360,19 @@ ::p3_main( infrastructure, history_only, lookup_tables, + temporaries, workspace_mgr, nj, nk, p3constants); +#else + return p3_main_internal(runtime_options, + prognostic_state, + diagnostic_inputs, + diagnostic_outputs, + infrastructure, + history_only, + lookup_tables, + workspace_mgr, + nj, nk, p3constants); #endif } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 3a5218f5c001..791538d67dcb 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -246,6 +246,45 @@ struct Functions view_dnu_table dnu_table_vals; }; +#ifdef SCREAM_P3_SMALL_KERNELS + struct P3Temporaries { + P3Temporaries() = default; + // shape parameter of rain + view_2d mu_r; + // temperature at the beginning of the microphysics step [K] + view_2d T_atm; + // 2D size distribution and fallspeed parameters + view_2d lamr, logn0r, nu; + view_2d cdist, cdist1, cdistr; + // Variables needed for in-cloud calculations + // Inverse cloud fractions (1/cld) + view_2d inv_cld_frac_i, inv_cld_frac_l, inv_cld_frac_r; + // In cloud mass-mixing ratios + view_2d qc_incld, qr_incld, qi_incld, qm_incld; + // In cloud number concentrations + view_2d nc_incld, nr_incld, ni_incld, bm_incld; + // Other + view_2d inv_dz, inv_rho, ze_ice, ze_rain; + view_2d prec, rho, rhofacr, rhofaci; + view_2d acn, qv_sat_l, qv_sat_i, sup; + view_2d qv_supersat_i, tmparr2, exner; + view_2d diag_equiv_reflectivity, diag_vm_qi, diag_diam_qi; + view_2d pratot, prctot; + // p3_tend_out, may not need these + view_2d qtend_ignore, ntend_ignore; + // Variables still used in F90 but removed from C++ interface + view_2d mu_c, lamc; + view_2d qr_evap_tend; + // cloud sedimentation + view_2d v_qc, v_nc, flux_qx, flux_nx; + // ice sedimentation + view_2d v_qit, v_nit, flux_nit, flux_bir; + view_2d flux_qir, flux_qit; + // rain sedimentation + view_2d v_qr, v_nr; + }; +#endif + // -- Table3 -- struct Table3 { @@ -1301,6 +1340,9 @@ struct Functions const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + const P3Temporaries& temporaries, +#endif const WorkspaceManager& workspace_mgr, Int nj, // number of columns Int nk, // number of vertical cells per column @@ -1328,6 +1370,7 @@ struct Functions const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, + const P3Temporaries& temporaries, const WorkspaceManager& workspace_mgr, Int nj, // number of columns Int nk, // number of vertical cells per column diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index eff3102f0e44..b8f6e826fc14 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -2054,6 +2054,36 @@ Int p3_main_f( P3F::P3HistoryOnly history_only{liq_ice_exchange_d, vap_liq_exchange_d, vap_ice_exchange_d}; + const Int nk_pack = ekat::npack(nk); +#ifdef SCREAM_P3_SMALL_KERNELS + view_2d + mu_r("mu_r", nj, nk_pack), T_atm("T_atm", nj, nk_pack), lamr("lamr", nj, nk_pack), logn0r("logn0r", nj, nk_pack), nu("nu", nj, nk_pack), + cdist("cdist", nj, nk_pack), cdist1("cdist1", nj, nk_pack), cdistr("cdistr", nj, nk_pack), inv_cld_frac_i("inv_cld_frac_i", nj, nk_pack), + inv_cld_frac_l("inv_cld_frac_l", nj, nk_pack), inv_cld_frac_r("inv_cld_frac_r", nj, nk_pack), qc_incld("qc_incld", nj, nk_pack), + qr_incld("qr_incld", nj, nk_pack), qi_incld("qi_incld", nj, nk_pack), qm_incld("qm_incld", nj, nk_pack), nc_incld("nc_incld", nj, nk_pack), + nr_incld("nr_incld", nj, nk_pack), ni_incld("ni_incld", nj, nk_pack), bm_incld("bm_incld", nj, nk_pack), inv_dz("inv_dz", nj, nk_pack), + inv_rho("inv_rho", nj, nk_pack), ze_ice("ze_ice", nj, nk_pack), ze_rain("ze_rain", nj, nk_pack), prec("prec", nj, nk_pack), + rho("rho", nj, nk_pack), rhofacr("rhofacr", nj, nk_pack), rhofaci("rhofaci", nj, nk_pack), acn("acn", nj, nk_pack), qv_sat_l("qv_sat", nj, nk_pack), + qv_sat_i("qv_sat_i", nj, nk_pack), sup("sup", nj, nk_pack), qv_supersat_i("qv_supersat", nj, nk_pack), tmparr2("tmparr2", nj, nk_pack), + exner("exner", nj, nk_pack), diag_equiv_reflectivity("diag_equiv_ref", nj, nk_pack), diag_vm_qi("diag_vm_qi", nj, nk_pack), + diag_diam_qi("diag_diam_qi", nj, nk_pack), pratot("pratot", nj, nk_pack), prctot("prctot", nj, nk_pack), qtend_ignore("qtend_ignore", nj, nk_pack), + ntend_ignore("ntend_ignore", nj, nk_pack), mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), qr_evap_tend("qr_evap_tend", nj, nk_pack), + v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), v_qit("v_qit", nj, nk_pack), + v_nit("v_nit", nj, nk_pack), flux_nit("flux_nit", nj, nk_pack), flux_bir("flux_bir", nj, nk_pack), flux_qir("flux_qir", nj, nk_pack), + flux_qit("flux_qit", nj, nk_pack), v_qr("v_qr", nj, nk_pack), v_nr("v_nr", nj, nk_pack); + + P3F::P3Temporaries temporaries{ + mu_r, T_atm, lamr, logn0r, nu, cdist, cdist1, cdistr, inv_cld_frac_i, + inv_cld_frac_l, inv_cld_frac_r, qc_incld, qr_incld, qi_incld, qm_incld, + nc_incld, nr_incld, ni_incld, bm_incld, inv_dz, inv_rho, ze_ice, ze_rain, + prec, rho, rhofacr, rhofaci, acn, qv_sat_l, qv_sat_i, sup, qv_supersat_i, + tmparr2, exner, diag_equiv_reflectivity, diag_vm_qi, diag_diam_qi, + pratot, prctot, qtend_ignore, ntend_ignore, mu_c, lamc, qr_evap_tend, + v_qc, v_nc, flux_qx, flux_nx, v_qit, v_nit, flux_nit, flux_bir, flux_qir, + flux_qit, v_qr, v_nr + }; +#endif + // load tables view_1d_table mu_r_table_vals; view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; @@ -2068,12 +2098,15 @@ Int p3_main_f( P3F::P3Runtime runtime_options{740.0e3}; // Create local workspace - const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); ekat::WorkspaceManager workspace_mgr(nk_pack, 52, policy); auto elapsed_microsec = P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, nj, nk, physics::P3_Constants()); + history_only, lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + temporaries, +#endif + workspace_mgr, nj, nk, physics::P3_Constants()); Kokkos::parallel_for(nj, KOKKOS_LAMBDA(const Int& i) { precip_liq_surf_temp_d(0, i / Spack::n)[i % Spack::n] = precip_liq_surf_d(i); From 9438a980a0caebe562568beccb9b1c4b614a390f Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 3 Sep 2024 10:54:25 -0600 Subject: [PATCH 399/477] Remove latent_heat_* in/out variables Replace with constant scalars --- .../eamxx/src/physics/p3/CMakeLists.txt | 13 +- .../physics/p3/disp/p3_ice_sed_impl_disp.cpp | 7 +- .../src/physics/p3/disp/p3_main_impl_disp.cpp | 13 +- .../p3/disp/p3_main_impl_part1_disp.cpp | 11 +- .../p3/disp/p3_main_impl_part2_disp.cpp | 25 +- .../p3/disp/p3_main_impl_part3_disp.cpp | 10 +- .../src/physics/p3/eti/p3_get_latent_heat.cpp | 14 - .../p3/impl/p3_evaporate_rain_impl.hpp | 8 +- .../p3/impl/p3_get_latent_heat_impl.hpp | 24 -- .../p3_get_time_space_phys_variables_impl.hpp | 12 +- .../p3/impl/p3_ice_cldliq_wet_growth_impl.hpp | 8 +- .../physics/p3/impl/p3_ice_melting_impl.hpp | 8 +- .../src/physics/p3/impl/p3_ice_sed_impl.hpp | 6 +- .../p3_ice_supersat_conservation_impl.hpp | 7 +- .../src/physics/p3/impl/p3_main_impl.hpp | 17 +- .../physics/p3/impl/p3_main_impl_part1.hpp | 13 +- .../physics/p3/impl/p3_main_impl_part2.hpp | 35 +- .../physics/p3/impl/p3_main_impl_part3.hpp | 10 +- .../p3_prevent_liq_supersaturation_impl.hpp | 20 +- .../p3/impl/p3_update_prognostics_impl.hpp | 15 +- .../eamxx/src/physics/p3/p3_functions.hpp | 39 +-- .../eamxx/src/physics/p3/p3_functions_f90.cpp | 309 ++++++++---------- .../eamxx/src/physics/p3/p3_functions_f90.hpp | 10 +- .../eamxx/src/physics/p3/tests/CMakeLists.txt | 1 - .../p3/tests/p3_evaporate_rain_unit_tests.cpp | 52 +-- .../tests/p3_get_latent_heat_unit_tests.cpp | 17 +- .../p3_ice_cldliq_wet_growth_unit_tests.cpp | 47 +-- .../p3/tests/p3_ice_melting_unit_tests.cpp | 46 +-- .../p3/tests/p3_ice_sed_unit_tests.cpp | 10 +- .../p3_ice_supersat_conservation_tests.cpp | 12 +- .../physics/p3/tests/p3_main_unit_tests.cpp | 48 ++- .../p3_prevent_liq_supersaturation_tests.cpp | 29 +- .../src/physics/p3/tests/p3_unit_tests.cpp | 125 ++++--- 33 files changed, 477 insertions(+), 544 deletions(-) delete mode 100644 components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp delete mode 100644 components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index fc0293cb35d0..a1dfc946267d 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -45,7 +45,6 @@ if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos eti/p3_ice_relaxation_timescale.cpp eti/p3_ice_nucleation.cpp eti/p3_ice_cldliq_wet_growth.cpp - eti/p3_get_latent_heat.cpp eti/p3_check_values.cpp eti/p3_incloud_mixingratios.cpp eti/p3_subgrid_variance_scaling.cpp @@ -63,13 +62,13 @@ endif() # List of dispatch source files if monolithic kernels are off set(P3_SK_SRCS - disp/p3_check_values_impl_disp.cpp - disp/p3_ice_sed_impl_disp.cpp - disp/p3_main_impl_part1_disp.cpp + disp/p3_check_values_impl_disp.cpp + disp/p3_ice_sed_impl_disp.cpp + disp/p3_main_impl_part1_disp.cpp disp/p3_main_impl_part3_disp.cpp - disp/p3_cloud_sed_impl_disp.cpp - disp/p3_main_impl_disp.cpp - disp/p3_main_impl_part2_disp.cpp + disp/p3_cloud_sed_impl_disp.cpp + disp/p3_main_impl_disp.cpp + disp/p3_main_impl_part2_disp.cpp disp/p3_rain_sed_impl_disp.cpp ) diff --git a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp index 0d2aad5247dd..f1e84750c250 100644 --- a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp @@ -46,8 +46,8 @@ ::ice_sedimentation_disp( // Ice sedimentation: (adaptive substepping) ice_sedimentation( - ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(cld_frac_i, i), - ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, + ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(cld_frac_i, i), + ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, ekat::subview(qi, i), ekat::subview(qi_incld, i), ekat::subview(ni, i), ekat::subview(ni_incld, i), ekat::subview(qm, i), ekat::subview(qm_incld, i), ekat::subview(bm, i), ekat::subview(bm_incld, i), ekat::subview(qi_tend, i), ekat::subview(ni_tend, i), ice_table_vals, precip_ice_surf(i), p3constants); @@ -60,7 +60,6 @@ void Functions ::homogeneous_freezing_disp( const uview_2d& T_atm, const uview_2d& inv_exner, - const uview_2d& latent_heat_fusion, const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_2d& qc, const uview_2d& nc, @@ -89,7 +88,7 @@ ::homogeneous_freezing_disp( // homogeneous freezing of cloud and rain homogeneous_freezing( - ekat::subview(T_atm, i), ekat::subview(inv_exner, i), ekat::subview(latent_heat_fusion, i), team, nk, ktop, kbot, kdir, + ekat::subview(T_atm, i), ekat::subview(inv_exner, i), team, nk, ktop, kbot, kdir, ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(th_atm, i)); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index 0f125b7ae960..b79b4f6d072c 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -115,10 +115,6 @@ ::p3_main_internal_disp( { using ExeSpace = typename KT::ExeSpace; - view_2d latent_heat_sublim("latent_heat_sublim", nj, nk), latent_heat_vapor("latent_heat_vapor", nj, nk), latent_heat_fusion("latent_heat_fusion", nj, nk); - - get_latent_heat(nj, nk, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion); - const Int nk_pack = ekat::npack(nk); // load constants into local vars @@ -244,7 +240,7 @@ ::p3_main_internal_disp( p3_main_part1_disp( nj, nk, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, pres, dpres, dz, nc_nuceat_tend, nccn_prescribed, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, - inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, + inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, @@ -259,8 +255,7 @@ ::p3_main_internal_disp( lookup_tables.revap_table_vals, pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - qv, th, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, - latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, + qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, @@ -301,7 +296,7 @@ ::p3_main_internal_disp( // homogeneous freezing f cloud and rain homogeneous_freezing_disp( - T_atm, inv_exner, latent_heat_fusion, nj, nk, ktop, kbot, kdir, qc, nc, qr, nr, qi, + T_atm, inv_exner, nj, nk, ktop, kbot, kdir, qc, nc, qr, nr, qi, ni, qm, bm, th, nucleationPossible, hydrometeorsPresent); // @@ -311,7 +306,7 @@ ::p3_main_internal_disp( p3_main_part3_disp( nj, nk_pack, runtime_options.max_total_ni, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, rho, inv_rho, rhofaci, qv, th, qc, nc, qr, nr, qi, ni, - qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, + qm, bm, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent, p3constants); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp index 3dae805255b0..28db7a6348dc 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp @@ -31,9 +31,6 @@ ::p3_main_part1_disp( const uview_2d& inv_cld_frac_l, const uview_2d& inv_cld_frac_i, const uview_2d& inv_cld_frac_r, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& T_atm, const uview_2d& rho, const uview_2d& inv_rho, @@ -73,17 +70,17 @@ ::p3_main_part1_disp( policy, KOKKOS_LAMBDA(const MemberType& team) { const Int i = team.league_rank(); - + p3_main_part1( team, nk, predictNc, prescribedCCN, dt, ekat::subview(pres, i), ekat::subview(dpres, i), ekat::subview(dz, i), ekat::subview(nc_nuceat_tend, i), ekat::subview(nccn_prescribed, i), ekat::subview(inv_exner, i), ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), - ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), - ekat::subview(latent_heat_fusion, i), ekat::subview(T_atm, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), + ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), + ekat::subview(T_atm, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(qc_incld, i), ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), - ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), + ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), nucleationPossible(i), hydrometeorsPresent(i), p3constants); }); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 2b619d54bf33..5aa32bf817f5 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -59,9 +59,6 @@ ::p3_main_part2_disp( const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& qc_incld, const uview_2d& qr_incld, const uview_2d& qi_incld, @@ -104,25 +101,25 @@ ::p3_main_part2_disp( const Int i = team.league_rank(); if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { - return; + return; } // ------------------------------------------------------------------------------------------ // main k-loop (for processes): p3_main_part2( team, nk_pack, max_total_ni, predictNc, do_prescribed_CCN, dt, inv_dt, - dnu_table_vals, ice_table_vals, collect_table_vals, revap_table_vals, + dnu_table_vals, ice_table_vals, collect_table_vals, revap_table_vals, ekat::subview(pres, i), ekat::subview(dpres, i), ekat::subview(dz, i), ekat::subview(nc_nuceat_tend, i), ekat::subview(inv_exner, i), - ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), - ekat::subview(ni_activated, i), ekat::subview(inv_qc_relvar, i), ekat::subview(cld_frac_i, i), ekat::subview(cld_frac_l, i), - ekat::subview(cld_frac_r, i), ekat::subview(qv_prev, i), ekat::subview(t_prev, i), ekat::subview(T_atm, i), ekat::subview(rho, i), - ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), - ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), - ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), - ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), ekat::subview(latent_heat_fusion, i), ekat::subview(qc_incld, i), - ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), + ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), + ekat::subview(ni_activated, i), ekat::subview(inv_qc_relvar, i), ekat::subview(cld_frac_i, i), ekat::subview(cld_frac_l, i), + ekat::subview(cld_frac_r, i), ekat::subview(qv_prev, i), ekat::subview(t_prev, i), ekat::subview(T_atm, i), ekat::subview(rho, i), + ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), + ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), + ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), + ekat::subview(qc_incld, i), + ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(cdist, i), - ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), + ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), ekat::subview(qv2qi_depos_tend, i), ekat::subview(precip_total_tend, i), ekat::subview(nevapr, i), ekat::subview(qr_evap_tend, i), ekat::subview(vap_liq_exchange, i), ekat::subview(vap_ice_exchange, i), ekat::subview(liq_ice_exchange, i), ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk, p3constants); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp index 6d591a7e9328..fd41fb5f0fe1 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp @@ -38,8 +38,6 @@ ::p3_main_part3_disp( const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, const uview_2d& mu_c, const uview_2d& nu, const uview_2d& lamc, @@ -77,11 +75,11 @@ ::p3_main_part3_disp( // p3_main_part3( team, nk_pack, max_total_ni, dnu_table_vals, ice_table_vals, ekat::subview(inv_exner, i), ekat::subview(cld_frac_l, i), ekat::subview(cld_frac_r, i), - ekat::subview(cld_frac_i, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(qv, i), - ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), - ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), + ekat::subview(cld_frac_i, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(qv, i), + ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), + ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), - ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), + ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), ekat::subview(diag_diam_qi, i), ekat::subview(rho_qi, i), ekat::subview(diag_equiv_reflectivity, i), ekat::subview(diag_eff_radius_qc, i), ekat::subview(diag_eff_radius_qr, i), p3constants); diff --git a/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp b/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp deleted file mode 100644 index 87e41c934587..000000000000 --- a/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "p3_get_latent_heat_impl.hpp" - -namespace scream { -namespace p3 { - -/* - * Explicit instantiation for doing conservation functions on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace p3 -} // namespace scream diff --git a/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp index 5455bfaa4c14..5560fcfe9123 100644 --- a/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp @@ -69,7 +69,7 @@ ::evaporate_rain( const Spack& cld_frac_l, const Spack& cld_frac_r, const Spack& qv, const Spack& qv_prev, const Spack& qv_sat_l, const Spack& qv_sat_i, const Spack& ab, const Spack& abi, const Spack& epsr, const Spack& epsi_tot, const Spack& t_atm, const Spack& t_atm_prev, - const Spack& latent_heat_sublim, const Spack& dqsdt, const Scalar& dt, + const Spack& dqsdt, const Scalar& dt, Spack& qr2qv_evap_tend, Spack& nr_evap_tend, const Smask& context) { @@ -92,6 +92,8 @@ ::evaporate_rain( constexpr Scalar QSMALL = C::QSMALL; constexpr Scalar Tmelt = C::Tmelt; constexpr Scalar inv_cp = 1/C::Cpair; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //Compute absolute supersaturation. //Ignore the difference between clear-sky and cell-ave qv and T @@ -133,9 +135,9 @@ ::evaporate_rain( const Smask not_freezing = !is_freezing && context; Spack eps_eff, A_c; if (is_freezing.any()){ - eps_eff.set(is_freezing,epsr + epsi_tot*(1 + latent_heat_sublim*inv_cp*dqsdt)/abi); + eps_eff.set(is_freezing,epsr + epsi_tot*(1 + (latvap+latice)*inv_cp*dqsdt)/abi); A_c.set(is_freezing,(qv - qv_prev)*inv_dt - dqsdt*(t_atm-t_atm_prev)*inv_dt - - (qv_sat_l - qv_sat_i)*(1 + latent_heat_sublim*inv_cp*dqsdt)/abi*epsi_tot ); + - (qv_sat_l - qv_sat_i)*(1 + (latvap+latice)*inv_cp*dqsdt)/abi*epsi_tot ); } if (not_freezing.any()){ eps_eff.set(not_freezing,epsr); diff --git a/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp deleted file mode 100644 index 439a1e2c84d6..000000000000 --- a/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef P3_GET_LATENT_HEAT_IMPL_HPP -#define P3_GET_LATENT_HEAT_IMPL_HPP - -#include "p3_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace p3 { - -template -void Functions -::get_latent_heat(const Int& nj, const Int& nk, view_2d& v, view_2d& s, view_2d& f) -{ - constexpr Scalar latvap = C::LatVap; - constexpr Scalar latice = C::LatIce; - - Kokkos::deep_copy(v, latvap); - Kokkos::deep_copy(s, latvap + latice); - Kokkos::deep_copy(f, latice); -} - -} // namespace p3 -} // namespace scream - -#endif // P3_GET_LATENT_HEAT_IMPL_HPP diff --git a/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp index f46c4aaebb5f..ea42d7995ba1 100644 --- a/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp @@ -11,7 +11,7 @@ template KOKKOS_FUNCTION void Functions ::get_time_space_phys_variables( - const Spack& T_atm, const Spack& pres, const Spack& rho, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, + const Spack& T_atm, const Spack& pres, const Spack& rho, const Spack& qv_sat_l, const Spack& qv_sat_i, Spack& mu, Spack& dv, Spack& sc, Spack& dqsdt, Spack& dqsidt, Spack& ab, Spack& abi, Spack& kap, Spack& eii, const Smask& context) @@ -23,15 +23,17 @@ ::get_time_space_phys_variables( constexpr Scalar RV = C::RV; constexpr Scalar INV_CP = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; constexpr Scalar tval1 = 253.15; constexpr Scalar tval2 = 273.15; constexpr Scalar dtval = 20; //this is tval2-tval1, but specifying here as int to be BFB with F90. const auto dum = 1/(RV*square(T_atm)); - dqsdt.set(context, latent_heat_vapor*qv_sat_l*dum); - dqsidt.set(context, latent_heat_sublim*qv_sat_i*dum); - ab.set(context, 1+dqsdt*latent_heat_vapor*INV_CP); - abi.set(context, 1+dqsidt*latent_heat_sublim*INV_CP); + dqsdt.set(context, latvap*qv_sat_l*dum); + dqsidt.set(context, (latvap+latice)*qv_sat_i*dum); + ab.set(context, 1+dqsdt*latvap*INV_CP); + abi.set(context, 1+dqsidt*(latvap+latice)*INV_CP); kap.set(context, sp(1.414e+3)*mu); //very simple temperature dependent aggregation efficiency diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp index cd8d9ce7ef0e..319d10f64ef7 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp @@ -13,7 +13,7 @@ KOKKOS_FUNCTION void Functions ::ice_cldliq_wet_growth( const Spack& rho, const Spack& temp, const Spack& pres, const Spack& rhofaci, const Spack& table_val_qi2qr_melting, - const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, const Spack& dv, + const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& kap, const Spack& mu, const Spack& sc, const Spack& qv, const Spack& qc_incld, const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Smask& log_wetgrowth, Spack& qr2qi_collect_tend, Spack& qc2qi_collect_tend, Spack& qc_growth_rate, Spack& nr_ice_shed_tend, Spack& qc2qr_ice_shed_tend, const Smask& context) @@ -26,6 +26,8 @@ ::ice_cldliq_wet_growth( constexpr Scalar zero = C::ZERO; constexpr Scalar one = C::ONE; constexpr Scalar cpw = C::CpLiq; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; const auto t_is_negative = temp < tmelt; const auto qi_incld_ge_small = qi_incld >= qsmall; @@ -46,8 +48,8 @@ ::ice_cldliq_wet_growth( qc_growth_rate.set(any_if, ((table_val_qi2qr_melting+table_val_qi2qr_vent_melt*cbrt(sc)*sqrt(rhofaci*rho/mu))* - twopi*(rho*latent_heat_vapor*dv*(qsat0-qv)-(temp-tmelt)*kap)/ - (latent_heat_fusion+cpw*(temp-tmelt)))*ni_incld); + twopi*(rho*latvap*dv*(qsat0-qv)-(temp-tmelt)*kap)/ + (latice+cpw*(temp-tmelt)))*ni_incld); qc_growth_rate.set(any_if, max(qc_growth_rate, zero)); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp index dc6c8d16256f..a4b86806ad0c 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp @@ -13,7 +13,7 @@ KOKKOS_FUNCTION void Functions ::ice_melting( const Spack& rho, const Spack& T_atm, const Spack& pres, const Spack& rhofaci, - const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, + const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& sc, const Spack& mu, const Spack& kap, const Spack& qv, const Spack& qi_incld, const Spack& ni_incld, Spack& qi2qr_melt_tend, Spack& ni2nr_melt_tend, const Smask& context) @@ -29,6 +29,8 @@ ::ice_melting( const auto Pi = C::Pi; const auto QSMALL = C::QSMALL; const auto Tmelt = C::Tmelt; + const auto latvap = C::LatVap; + const auto latice = C::LatIce; //Find cells above freezing AND which have ice const auto has_melt_qi = (qi_incld >= QSMALL ) && (T_atm > Tmelt) && context; @@ -38,8 +40,8 @@ ::ice_melting( const auto qsat0 = physics::qv_sat_dry(Spack(Tmelt), pres, false, context, physics::MurphyKoop, "p3::ice_melting"); //"false" here means NOT saturation w/ respect to ice. qi2qr_melt_tend.set(has_melt_qi, ( (table_val_qi2qr_melting+table_val_qi2qr_vent_melt*cbrt(sc)*sqrt(rhofaci*rho/mu)) - *((T_atm-Tmelt)*kap-rho*latent_heat_vapor*dv*(qsat0-qv)) - * 2 * Pi /latent_heat_fusion)*ni_incld ); + *((T_atm-Tmelt)*kap-rho*latvap*dv*(qsat0-qv)) + * 2 * Pi /latice)*ni_incld ); //make sure qi2qr_melt_tend is always negative qi2qr_melt_tend = max(qi2qr_melt_tend, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp index 29232c69e68f..1273d7bbc206 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp @@ -210,7 +210,6 @@ void Functions ::homogeneous_freezing( const uview_1d& T_atm, const uview_1d& inv_exner, - const uview_1d& latent_heat_fusion, const MemberType& team, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_1d& qc, @@ -228,6 +227,7 @@ ::homogeneous_freezing( constexpr Scalar T_homogfrz = C::T_homogfrz; constexpr Scalar inv_rho_rimeMax = C::INV_RHO_RIMEMAX; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latice = C::LatIce; const Int kmin_scalar = ( kdir == 1 ? kbot : ktop); const Int kmax_scalar = ( kdir == 1 ? ktop : kbot); @@ -254,13 +254,13 @@ ::homogeneous_freezing( qi(pk).set(qc_ge_small, qi(pk) + Qc_nuc); bm(pk).set(qc_ge_small, bm(pk) + Qc_nuc*inv_rho_rimeMax); ni(pk).set(qc_ge_small, ni(pk) + Nc_nuc); - th_atm(pk).set (qc_ge_small, th_atm(pk) + inv_exner(pk)*Qc_nuc*latent_heat_fusion(pk)*inv_cp); + th_atm(pk).set (qc_ge_small, th_atm(pk) + inv_exner(pk)*Qc_nuc*latice*inv_cp); qm(pk).set(qr_ge_small, qm(pk) + Qr_nuc); qi(pk).set(qr_ge_small, qi(pk) + Qr_nuc); bm(pk).set(qr_ge_small, bm(pk) + Qr_nuc*inv_rho_rimeMax); ni(pk).set(qr_ge_small, ni(pk) + Nr_nuc); - th_atm(pk).set (qr_ge_small, th_atm(pk) + inv_exner(pk)*Qr_nuc*latent_heat_fusion(pk)*inv_cp); + th_atm(pk).set (qr_ge_small, th_atm(pk) + inv_exner(pk)*Qr_nuc*latice*inv_cp); qc(pk).set(qc_ge_small, 0); nc(pk).set(qc_ge_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp index 1bed57bad630..091aa20f58d4 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp @@ -13,11 +13,14 @@ namespace p3 { template KOKKOS_FUNCTION -void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& qv2qi_nucleat_tend, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& latent_heat_sublim, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context) +void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& qv2qi_nucleat_tend, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar cp = C::CP; constexpr Scalar rv = C::RH2O; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + constexpr Scalar latsublim2 = (latvap+latice)*(latvap+latice); const auto qv_sink = qv2qi_vapdep_tend + qv2qi_nucleat_tend; // in [kg/kg] cell-avg values @@ -25,7 +28,7 @@ void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& if (mask.any()) { // --- Available water vapor for deposition/nucleation auto qv_avail = (qv + (qi2qv_sublim_tend+qr2qv_evap_tend)*dt - qv_sat_i) / - (1 + square(latent_heat_sublim)*qv_sat_i / (cp*rv*square(t_atm)) ) / dt; + (1 + latsublim2*qv_sat_i / (cp*rv*square(t_atm)) ) / dt; // --- Only excess water vapor can be limited qv_avail = max(qv_avail, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index 270042c99986..a36a79364044 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -88,10 +88,6 @@ ::p3_main_internal( { using ExeSpace = typename KT::ExeSpace; - view_2d latent_heat_sublim("latent_heat_sublim", nj, nk), latent_heat_vapor("latent_heat_vapor", nj, nk), latent_heat_fusion("latent_heat_fusion", nj, nk); - - get_latent_heat(nj, nk, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion); - const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); @@ -202,9 +198,6 @@ ::p3_main_internal( const auto oliq_ice_exchange = ekat::subview(history_only.liq_ice_exchange, i); const auto ovap_liq_exchange = ekat::subview(history_only.vap_liq_exchange, i); const auto ovap_ice_exchange = ekat::subview(history_only.vap_ice_exchange, i); - const auto olatent_heat_vapor = ekat::subview(latent_heat_vapor, i); - const auto olatent_heat_sublim = ekat::subview(latent_heat_sublim, i); - const auto olatent_heat_fusion = ekat::subview(latent_heat_fusion, i); const auto oqv_prev = ekat::subview(diagnostic_inputs.qv_prev, i); const auto ot_prev = ekat::subview(diagnostic_inputs.t_prev, i); @@ -232,7 +225,7 @@ ::p3_main_internal( p3_main_part1( team, nk, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, opres, odpres, odz, onc_nuceat_tend, onccn_prescribed, oinv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, - inv_cld_frac_r, olatent_heat_vapor, olatent_heat_sublim, olatent_heat_fusion, + inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, @@ -251,8 +244,8 @@ ::p3_main_internal( lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, lookup_tables.revap_table_vals, opres, odpres, odz, onc_nuceat_tend, oinv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, oni_activated, oinv_qc_relvar, ocld_frac_i, ocld_frac_l, ocld_frac_r, oqv_prev, ot_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, olatent_heat_vapor, - olatent_heat_sublim, olatent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, + oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, + qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, oqv2qi_depos_tend, oprecip_total_tend, onevapr, qr_evap_tend, ovap_liq_exchange, ovap_ice_exchange, oliq_ice_exchange, @@ -295,7 +288,7 @@ ::p3_main_internal( // homogeneous freezing of cloud and rain homogeneous_freezing( - T_atm, oinv_exner, olatent_heat_fusion, team, nk, ktop, kbot, kdir, oqc, onc, oqr, onr, oqi, + T_atm, oinv_exner, team, nk, ktop, kbot, kdir, oqc, onc, oqr, onr, oqi, oni, oqm, obm, oth); // @@ -305,7 +298,7 @@ ::p3_main_internal( p3_main_part3( team, nk_pack, runtime_options.max_total_ni, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, oinv_exner, ocld_frac_l, ocld_frac_r, ocld_frac_i, rho, inv_rho, rhofaci, oqv, oth, oqc, onc, oqr, onr, oqi, oni, - oqm, obm, olatent_heat_vapor, olatent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, + oqm, obm, mu_c, nu, lamc, mu_r, lamr, ovap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, odiag_eff_radius_qi, diag_diam_qi, orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr, p3constants); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp index 439b0544adf9..6771a48d4c68 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp @@ -34,9 +34,6 @@ ::p3_main_part1( const uview_1d& inv_cld_frac_l, const uview_1d& inv_cld_frac_i, const uview_1d& inv_cld_frac_r, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& T_atm, const uview_1d& rho, const uview_1d& inv_rho, @@ -80,6 +77,8 @@ ::p3_main_part1( constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; const Scalar p3_spa_to_nc = p3constants.p3_spa_to_nc; @@ -124,7 +123,7 @@ ::p3_main_part1( auto drymass = qc(k) < qsmall; auto not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qc(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qc(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qc(k) * latvap * inv_cp); qc(k).set(drymass, 0); nc(k).set(drymass, 0); if ( not_drymass.any() ) { @@ -147,7 +146,7 @@ ::p3_main_part1( drymass = qr(k) < qsmall; not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qr(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qr(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qr(k) * latvap * inv_cp); qr(k).set(drymass, 0); nr(k).set(drymass, 0); if ( not_drymass.any() ) { @@ -157,7 +156,7 @@ ::p3_main_part1( drymass = (qi(k) < qsmall || (qi(k) < 1.e-8 && qv_supersat_i(k) < -0.1)); not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qi(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_sublim(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * (latvap+latice) * inv_cp); qi(k).set(drymass, 0); ni(k).set(drymass, 0); qm(k).set(drymass, 0); @@ -168,7 +167,7 @@ ::p3_main_part1( drymass = (qi(k) >= qsmall && qi(k) < 1.e-8 && T_atm(k) >= T_zerodegc); qr(k).set(drymass, qr(k) + qi(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_fusion(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latice * inv_cp); qi(k).set(drymass, 0); ni(k).set(drymass, 0); qm(k).set(drymass, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp index c32fb7202ebb..d19dc579b7e9 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp @@ -65,9 +65,6 @@ ::p3_main_part2( const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& qc_incld, const uview_1d& qr_incld, const uview_1d& qi_incld, @@ -104,6 +101,8 @@ ::p3_main_part2( constexpr Scalar f2r = C::f2r; constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; team.team_barrier(); hydrometeorsPresent = false; @@ -115,12 +114,12 @@ ::p3_main_part2( //compute mask to identify padded values in packs, which shouldn't be used in calculations const auto range_pack = ekat::range(k*Spack::n); const auto range_mask = range_pack < nk; - + // if relatively dry and no hydrometeors at this level, skip to end of k-loop (i.e. skip this level) const auto skip_all = ( !range_mask || (qc(k)= qsmall && not_skip_all; qv(k).set(qc_small, qv(k) + qc(k)); - th_atm(k).set(qc_small, th_atm(k) - inv_exner(k) * qc(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(qc_small, th_atm(k) - inv_exner(k) * qc(k) * latvap * inv_cp); qc(k).set(qc_small, 0); nc(k).set(qc_small, 0); @@ -464,7 +463,7 @@ ::p3_main_part2( } qv(k).set(qr_small, qv(k) + qr(k)); - th_atm(k).set(qr_small, th_atm(k) - inv_exner(k) * qr(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(qr_small, th_atm(k) - inv_exner(k) * qr(k) * latvap * inv_cp); qr(k).set(qr_small, 0); nr(k).set(qr_small, 0); @@ -473,7 +472,7 @@ ::p3_main_part2( } qv(k).set(qi_small, qv(k) + qi(k)); - th_atm(k).set(qi_small, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_sublim(k) * inv_cp); + th_atm(k).set(qi_small, th_atm(k) - inv_exner(k) * qi(k) * (latvap+latice) * inv_cp); qi(k).set(qi_small, 0); ni(k).set(qi_small, 0); qm(k).set(qi_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp index 441bcb9feb9b..febb00ef4d31 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp @@ -41,8 +41,6 @@ ::p3_main_part3( const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, const uview_1d& mu_c, const uview_1d& nu, const uview_1d& lamc, @@ -63,6 +61,8 @@ ::p3_main_part3( constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; constexpr Scalar nsmall = C::NSMALL; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nk_pack), [&] (Int k) { @@ -93,7 +93,7 @@ ::p3_main_part3( } if (qc_small.any()) { qv(k) .set(qc_small, qv(k)+qc(k)); - th_atm(k) .set(qc_small, th_atm(k)-inv_exner(k)*qc(k)*latent_heat_vapor(k)*inv_cp); + th_atm(k) .set(qc_small, th_atm(k)-inv_exner(k)*qc(k)*latvap*inv_cp); vap_liq_exchange(k) .set(qc_small, vap_liq_exchange(k) - qc(k)); qc(k) .set(qc_small, 0); nc(k) .set(qc_small, 0); @@ -123,7 +123,7 @@ ::p3_main_part3( if (qr_small.any()) { qv(k) .set(qr_small, qv(k) + qr(k)); - th_atm(k) .set(qr_small, th_atm(k) - inv_exner(k)*qr(k)*latent_heat_vapor(k)*inv_cp); + th_atm(k) .set(qr_small, th_atm(k) - inv_exner(k)*qr(k)*latvap*inv_cp); vap_liq_exchange(k).set(qr_small, vap_liq_exchange(k) - qr(k)); qr(k) .set(qr_small, 0); nr(k) .set(qr_small, 0); @@ -185,7 +185,7 @@ ::p3_main_part3( ze_ice(k).set(qi_gt_small, ze_ice(k)*cld_frac_i(k)); qv(k).set(qi_small, qv(k) + qi(k)); - th_atm(k).set(qi_small, th_atm(k) - inv_exner(k)*qi(k)*latent_heat_sublim(k)*inv_cp); + th_atm(k).set(qi_small, th_atm(k) - inv_exner(k)*qi(k)*(latvap+latice)*inv_cp); qi(k).set(qi_small, 0); ni(k).set(qi_small, 0); qm(k).set(qi_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp index a9f265d8dcdd..10f728f1a079 100644 --- a/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp @@ -14,7 +14,7 @@ namespace p3 { template KOKKOS_FUNCTION -void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Scalar& dt, const Spack& qv2qi_vapdep_tend, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context) +void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Scalar& dt, const Spack& qv2qi_vapdep_tend, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context) // Note: context masks cells which are just padding for packs or which don't have any condensate worth // performing calculations on. { @@ -23,6 +23,8 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& constexpr Scalar inv_cp = C::INV_CP; constexpr Scalar rv = C::RV; constexpr Scalar qsmall = C::QSMALL; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; Spack qv_sinks, qv_sources, qv_endstep, T_endstep, A, frac; @@ -37,8 +39,8 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& //Actual qv and T after microphys step qv_endstep.set(has_sources,qv - qv_sinks*dt + qv_sources*dt); - T_endstep.set(has_sources,t_atm + ( (qv_sinks-qi2qv_sublim_tend)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend*latent_heat_vapor*inv_cp )*dt); + T_endstep.set(has_sources,t_atm + ( (qv_sinks-qi2qv_sublim_tend)*(latvap+latice)*inv_cp + - qr2qv_evap_tend*latvap*inv_cp )*dt); //qv we would have at end of step if we were saturated with respect to liquid const auto qsl = physics::qv_sat_dry(T_endstep,pres,false,has_sources,physics::MurphyKoop,"p3::prevent_liq_supersaturation"); //"false" means NOT sat w/ respect to ice @@ -46,18 +48,18 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& //The balance we seek is: // qv-qv_sinks*dt+qv_sources*frac*dt=qsl+dqsl_dT*(T correction due to conservation) // where the T correction for conservation is: - // dt*[latent_heat_sublim/cp*(qi2qv_sublim_tend-frac*qi2qv_sublim_tend) - // +latent_heat_vapor/cp*(qr2qv_evap_tend -frac*qr2qv_evap_tend)] - // =(1-frac)*dt/cp*(latent_heat_sublim*qi2qv_sublim_tend + latent_heat_vap*qr2qv_evap_tend). + // dt*[(latvap+latice)/cp*(qi2qv_sublim_tend-frac*qi2qv_sublim_tend) + // +latvap/cp*(qr2qv_evap_tend -frac*qr2qv_evap_tend)] + // =(1-frac)*dt/cp*((latvap+latice)*qi2qv_sublim_tend + latvap*qr2qv_evap_tend). // Note T correction is positive because frac *reduces* evaporative cooling. Note as well that // dqsl_dt comes from linearization of qsl around the end-of-step T computed before temperature // correction. dqsl_dt should be computed with respect to *liquid* even though frac also adjusts // sublimation because we want to be saturated with respect to liquid at the end of the step. - // dqsl_dt=Latent_heat_vapor*qsl/rv*T^2 following Clausius Clapeyron. Combining and solving for + // dqsl_dt=latvap*qsl/rv*T^2 following Clausius Clapeyron. Combining and solving for // frac yields: - A.set(has_sources,latent_heat_vapor*qsl*dt*inv_cp/(rv*T_endstep*T_endstep) - * (latent_heat_sublim*qi2qv_sublim_tend + latent_heat_vapor*qr2qv_evap_tend) ); + A.set(has_sources,latvap*qsl*dt*inv_cp/(rv*T_endstep*T_endstep) + * ((latvap+latice)*qi2qv_sublim_tend + latvap*qr2qv_evap_tend) ); frac.set(has_sources, (qsl-qv+qv_sinks*dt + A)/(qv_sources*dt + A) ); diff --git a/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp index 3f8bcab2cd51..86feaa04f499 100644 --- a/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp @@ -15,7 +15,7 @@ ::update_prognostic_ice( const Spack& qr2qi_immers_freeze_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nr_ice_shed_tend, const Spack& qi2qr_melt_tend, const Spack& ni2nr_melt_tend, const Spack& qi2qv_sublim_tend, const Spack& qv2qi_vapdep_tend, const Spack& qv2qi_nucleat_tend, const Spack& ni_nucleat_tend, const Spack& ni_selfcollect_tend, const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, - const Spack& inv_exner, const Spack& latent_heat_sublim, const Spack& latent_heat_fusion, const bool do_predict_nc, + const Spack& inv_exner, const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, const Scalar& nmltratio, const Spack& rho_qm_cloud, Spack& th_atm, Spack& qv, Spack& qi, Spack& ni, Spack& qm, Spack& bm, Spack& qc, Spack& nc, Spack& qr, Spack& nr, @@ -23,6 +23,8 @@ ::update_prognostic_ice( { constexpr Scalar QSMALL = C::QSMALL; constexpr Scalar INV_RHO_RIMEMAX = C::INV_RHO_RIMEMAX; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; qc.set(context, qc + (-qc2qi_hetero_freeze_tend-qc2qi_collect_tend-qc2qr_ice_shed_tend-qc2qi_berg_tend)*dt); if ( do_predict_nc ){ @@ -76,9 +78,9 @@ ::update_prognostic_ice( qv.set(context, qv + (-qv2qi_vapdep_tend+qi2qv_sublim_tend-qv2qi_nucleat_tend)*dt); constexpr Scalar INV_CP = C::INV_CP; - th_atm.set(context, th_atm + inv_exner * ((qv2qi_vapdep_tend - qi2qv_sublim_tend + qv2qi_nucleat_tend) * latent_heat_sublim * INV_CP + - (qr2qi_collect_tend + qc2qi_collect_tend + qc2qi_hetero_freeze_tend + qr2qi_immers_freeze_tend - - qi2qr_melt_tend + qc2qi_berg_tend) * latent_heat_fusion * INV_CP) * dt); + th_atm.set(context, th_atm + inv_exner * ((qv2qi_vapdep_tend - qi2qv_sublim_tend + qv2qi_nucleat_tend) * (latvap+latice) * INV_CP + + (qr2qi_collect_tend + qc2qi_collect_tend + qc2qi_hetero_freeze_tend + qr2qi_immers_freeze_tend - + qi2qr_melt_tend + qc2qi_berg_tend) * latice * INV_CP) * dt); } template @@ -88,13 +90,14 @@ ::update_prognostic_liquid( const Spack& qc2qr_accret_tend, const Spack& nc_accret_tend, const Spack& qc2qr_autoconv_tend,const Spack& nc2nr_autoconv_tend, const Spack& ncautr, const Spack& nc_selfcollect_tend, const Spack& qr2qv_evap_tend, const Spack& nr_evap_tend, const Spack& nr_selfcollect_tend, - const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Spack& latent_heat_vapor, + const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Scalar dt, Spack& th_atm, Spack& qv, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context) { constexpr Scalar NCCNST = C::NCCNST; constexpr int IPARAM = C::IPARAM; constexpr Scalar INV_CP = C::INV_CP; + constexpr Scalar latvap = C::LatVap; qc.set(context, qc + (-qc2qr_accret_tend-qc2qr_autoconv_tend)*dt); qr.set(context, qr + (qc2qr_accret_tend+qc2qr_autoconv_tend-qr2qv_evap_tend)*dt); @@ -115,7 +118,7 @@ ::update_prognostic_liquid( qv.set(context, qv + qr2qv_evap_tend *dt); - th_atm.set(context, th_atm + inv_exner*(-qr2qv_evap_tend * latent_heat_vapor * INV_CP) * dt); + th_atm.set(context, th_atm + inv_exner*(-qr2qv_evap_tend * latvap * INV_CP) * dt); } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 791538d67dcb..81a492688007 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -588,7 +588,6 @@ struct Functions static void homogeneous_freezing( const uview_1d& T_atm, const uview_1d& inv_exner, - const uview_1d& latent_heat_fusion, const MemberType& team, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_1d& qc, @@ -605,7 +604,6 @@ struct Functions static void homogeneous_freezing_disp( const uview_2d& T_atm, const uview_2d& inv_exner, - const uview_2d& latent_heat_fusion, const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_2d& qc, const uview_2d& nc, @@ -767,8 +765,8 @@ struct Functions const Spack& qr2qi_collect_tend, const Spack& nr_collect_tend, const Spack& qr2qi_immers_freeze_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nr_ice_shed_tend, const Spack& qi2qr_melt_tend, const Spack& ni2nr_melt_tend, const Spack& qi2qv_sublim_tend, const Spack& qv2qi_vapdep_tend, const Spack& qv2qi_nucleat_tend, const Spack& ni_nucleat_tend, const Spack& ni_selfcollect_tend, - const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, const Spack& inv_exner, const Spack& latent_heat_sublim, - const Spack& latent_heat_fusion, const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, + const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, const Spack& inv_exner, + const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, const Scalar& nmltratio, const Spack& rho_qm_cloud, Spack& th_atm, Spack& qv, Spack& qi, Spack& ni, Spack& qm, Spack& bm, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context = Smask(true)); @@ -776,7 +774,6 @@ struct Functions // TODO (comments) KOKKOS_FUNCTION static void get_time_space_phys_variables(const Spack& T_atm, const Spack& pres, const Spack& rho, - const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Spack& qv_sat_l, const Spack& qv_sat_i, Spack& mu, Spack& dv, Spack& sc, Spack& dqsdt, Spack& dqsidt, Spack& ab, Spack& abi, Spack& kap, Spack& eii, @@ -835,14 +832,14 @@ struct Functions const Spack& cld_frac_l, const Spack& cld_frac_r, const Spack& qv, const Spack& qv_prev, const Spack& qv_sat_l, const Spack& qv_sat_i, const Spack& ab, const Spack& abi, const Spack& epsr, const Spack & epsi_tot, const Spack& t, const Spack& t_prev, - const Spack& latent_heat_sublim, const Spack& dqsdt, const Scalar& dt, + const Spack& dqsdt, const Scalar& dt, Spack& qr2qv_evap_tend, Spack& nr_evap_tend, const Smask& context = Smask(true)); //get number and mass tendencies due to melting ice KOKKOS_FUNCTION static void ice_melting(const Spack& rho, const Spack& T_atm, const Spack& pres, const Spack& rhofaci, - const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, + const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& sc, const Spack& mu, const Spack& kap, const Spack& qv, const Spack& qi_incld, const Spack& ni_incld, Spack& qi2qr_melt_tend, Spack& ni2nr_melt_tend, const Smask& context = Smask(true)); @@ -852,7 +849,7 @@ struct Functions static void update_prognostic_liquid(const Spack& qc2qr_accret_tend, const Spack& nc_accret_tend, const Spack& qc2qr_autoconv_tend,const Spack& nc2nr_autoconv_tend, const Spack& ncautr, const Spack& nc_selfcollect_tend, const Spack& qr2qv_evap_tend, const Spack& nr_evap_tend, const Spack& nr_selfcollect_tend, - const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Spack& latent_heat_vapor, + const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Scalar dt, Spack& th_atm, Spack& qv, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context = Smask(true)); @@ -893,15 +890,12 @@ struct Functions KOKKOS_FUNCTION static void ice_cldliq_wet_growth(const Spack& rho, const Spack& temp, const Spack& pres, const Spack& rhofaci, const Spack& table_val_qi2qr_melting, - const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, const Spack& dv, + const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& kap, const Spack& mu, const Spack& sc, const Spack& qv, const Spack& qc_incld, const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Smask& log_wetgrowth, Spack& qr2qi_collect_tend, Spack& qc2qi_collect_tend, Spack& qc_growth_rate, Spack& nr_ice_shed_tend, Spack& qc2qr_ice_shed_tend, const Smask& context = Smask(true)); - // Note: not a kernel function - static void get_latent_heat(const Int& nj, const Int& nk, view_2d& v, view_2d& s, view_2d& f); - KOKKOS_FUNCTION static void check_values(const uview_1d& qv, const uview_1d& temp, const Int& ktop, const Int& kbot, const Int& timestepcount, const bool& force_abort, const Int& source_ind, const MemberType& team, @@ -993,9 +987,6 @@ struct Functions const uview_1d& inv_cld_frac_l, const uview_1d& inv_cld_frac_i, const uview_1d& inv_cld_frac_r, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& T_atm, const uview_1d& rho, const uview_1d& inv_rho, @@ -1044,9 +1035,6 @@ struct Functions const uview_2d& inv_cld_frac_l, const uview_2d& inv_cld_frac_i, const uview_2d& inv_cld_frac_r, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& T_atm, const uview_2d& rho, const uview_2d& inv_rho, @@ -1127,9 +1115,6 @@ struct Functions const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& qc_incld, const uview_1d& qr_incld, const uview_1d& qi_incld, @@ -1208,9 +1193,6 @@ struct Functions const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& qc_incld, const uview_2d& qr_incld, const uview_2d& qi_incld, @@ -1266,8 +1248,6 @@ struct Functions const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, const uview_1d& mu_c, const uview_1d& nu, const uview_1d& lamc, @@ -1309,8 +1289,6 @@ struct Functions const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, const uview_2d& mu_c, const uview_2d& nu, const uview_2d& lamc, @@ -1378,7 +1356,7 @@ struct Functions #endif KOKKOS_FUNCTION - static void ice_supersat_conservation(Spack& qidep, Spack& qinuc, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& latent_heat_sublim, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context = Smask(true)); + static void ice_supersat_conservation(Spack& qidep, Spack& qinuc, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context = Smask(true)); KOKKOS_FUNCTION static void nc_conservation(const Spack& nc, const Spack& nc_selfcollect_tend, const Real& dt, Spack& nc_collect_tend, Spack& nc2ni_immers_freeze_tend, Spack& nc_accret_tend, Spack& nc2nr_autoconv_tend, const Smask& context = Smask(true)); @@ -1390,7 +1368,7 @@ struct Functions static void ni_conservation(const Spack& ni, const Spack& ni_nucleat_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nc2ni_immers_freeze_tend, const Real& dt, Spack& ni2nr_melt_tend, Spack& ni_sublim_tend, Spack& ni_selfcollect_tend, const Smask& context = Smask(true)); KOKKOS_FUNCTION - static void prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Scalar& dt, const Spack& qidep, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context = Smask(true) ); + static void prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Scalar& dt, const Spack& qidep, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context = Smask(true) ); }; // struct Functions template @@ -1438,7 +1416,6 @@ void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, # include "p3_ice_melting_impl.hpp" # include "p3_calc_liq_relaxation_timescale_impl.hpp" # include "p3_ice_cldliq_wet_growth_impl.hpp" -# include "p3_get_latent_heat_impl.hpp" # include "p3_check_values_impl.hpp" # include "p3_incloud_mixingratios_impl.hpp" # include "p3_subgrid_variance_scaling_impl.hpp" diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index b8f6e826fc14..40d82d4f6529 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -1417,7 +1417,7 @@ void rain_sedimentation_f( void homogeneous_freezing_f( Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, + Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm) { using P3F = Functions; @@ -1440,31 +1440,31 @@ void homogeneous_freezing_f( const Int nk_pack = ekat::npack(nk); // Set up views - std::vector temp_d(HomogeneousFreezingData::NUM_ARRAYS); + std::vector temp_d(HomogeneousFreezingData::NUM_ARRAYS-1); - ekat::host_to_device({T_atm, inv_exner, latent_heat_fusion, qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, + ekat::host_to_device({T_atm, inv_exner, qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, temp_d); + int current_index = 0; view_1d - t_d (temp_d[0]), - inv_exner_d (temp_d[1]), - latent_heat_fusion_d (temp_d[2]), - qc_d (temp_d[3]), - nc_d (temp_d[4]), - qr_d (temp_d[5]), - nr_d (temp_d[6]), - qi_d (temp_d[7]), - ni_d (temp_d[8]), - qm_d (temp_d[9]), - bm_d (temp_d[10]), - th_atm_d (temp_d[11]); + t_d (temp_d[current_index++]), + inv_exner_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]); // Call core function from kernel auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { P3F::homogeneous_freezing( - t_d, inv_exner_d, latent_heat_fusion_d, + t_d, inv_exner_d, team, nk, ktop, kbot, kdir, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, th_atm_d); @@ -1476,37 +1476,6 @@ void homogeneous_freezing_f( ekat::device_to_host({qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, inout_views); } -void get_latent_heat_f(Int its, Int ite, Int kts, Int kte, Real* v, Real* s, Real* f) -{ - using P3F = Functions; - using Spack = typename P3F::Spack; - using view_2d = typename P3F::view_2d; - - EKAT_REQUIRE_MSG(kte >= kts, - "kte must be >= kts, kts=" << kts << " kte=" << kte); - - EKAT_REQUIRE_MSG(ite >= its, - "ite must be >= its, its=" << its << " ite=" << ite); - - kts -= 1; - kte -= 1; - its -= 1; - ite -= 1; - - Int nk = (kte - kts) + 1; - Int nj = (ite - its) + 1; - - // Set up views - view_2d v_d("v_d", nj, nk), - s_d("s_d", nj, nk), - f_d("f_d", nj, nk); - - P3F::get_latent_heat(nj, nk, v_d, s_d, f_d); - - std::vector out_views = {v_d, s_d, f_d}; - ekat::device_to_host({v, s, f}, nj, nk, out_views, true); -} - void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc) { @@ -1545,7 +1514,7 @@ void p3_main_part1_f( bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, + Real* inv_cld_frac_r, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, @@ -1576,7 +1545,7 @@ void p3_main_part1_f( ekat::host_to_device({pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, - acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, + acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, nccn_prescribed}, nk, temp_d); @@ -1609,18 +1578,15 @@ void p3_main_part1_f( ni_d (temp_d[25]), qm_d (temp_d[26]), bm_d (temp_d[27]), - latent_heat_vapor_d (temp_d[28]), - latent_heat_sublim_d (temp_d[29]), - latent_heat_fusion_d (temp_d[30]), - qc_incld_d (temp_d[31]), - qr_incld_d (temp_d[32]), - qi_incld_d (temp_d[33]), - qm_incld_d (temp_d[34]), - nc_incld_d (temp_d[35]), - nr_incld_d (temp_d[36]), - ni_incld_d (temp_d[37]), - bm_incld_d (temp_d[38]), - nccn_prescribed_d (temp_d[39]); + qc_incld_d (temp_d[28]), + qr_incld_d (temp_d[29]), + qi_incld_d (temp_d[30]), + qm_incld_d (temp_d[31]), + nc_incld_d (temp_d[32]), + nr_incld_d (temp_d[33]), + ni_incld_d (temp_d[34]), + bm_incld_d (temp_d[35]), + nccn_prescribed_d (temp_d[36]); // Call core function from kernel bview_1d bools_d("bools", 2); @@ -1630,7 +1596,7 @@ void p3_main_part1_f( P3F::p3_main_part1( team, nk, do_predict_nc, do_prescribed_CCN, dt, pres_d, dpres_d, dz_d, nc_nuceat_tend_d, nccn_prescribed_d, inv_exner_d, exner_d, inv_cld_frac_l_d, inv_cld_frac_i_d, - inv_cld_frac_r_d, latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, + inv_cld_frac_r_d, t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, @@ -1661,7 +1627,7 @@ void p3_main_part2_f( Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, + Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present) @@ -1688,82 +1654,80 @@ void p3_main_part2_f( const Real max_total_ni = 740.0e3; // Hard-code this value for F90 comparison // Set up views - std::vector temp_d(P3MainPart2Data::NUM_ARRAYS); + std::vector temp_d(P3MainPart2Data::NUM_ARRAYS-3); ekat::host_to_device({pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, + qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, prctot, qv_prev, t_prev }, nk, temp_d); + int current_index = 0; view_1d - pres_d (temp_d[0]), - dpres_d (temp_d[1]), - dz_d (temp_d[2]), - nc_nuceat_tend_d (temp_d[3]), - inv_exner_d (temp_d[4]), - exner_d (temp_d[5]), - inv_cld_frac_l_d (temp_d[6]), - inv_cld_frac_i_d (temp_d[7]), - inv_cld_frac_r_d (temp_d[8]), - ni_activated_d (temp_d[9]), - inv_qc_relvar_d (temp_d[10]), - cld_frac_i_d (temp_d[11]), - cld_frac_l_d (temp_d[12]), - cld_frac_r_d (temp_d[13]), - t_d (temp_d[14]), - rho_d (temp_d[15]), - inv_rho_d (temp_d[16]), - qv_sat_l_d (temp_d[17]), - qv_sat_i_d (temp_d[18]), - qv_supersat_i_d (temp_d[19]), - rhofacr_d (temp_d[20]), - rhofaci_d (temp_d[21]), - acn_d (temp_d[22]), - qv_d (temp_d[23]), - th_atm_d (temp_d[24]), - qc_d (temp_d[25]), - nc_d (temp_d[26]), - qr_d (temp_d[27]), - nr_d (temp_d[28]), - qi_d (temp_d[29]), - ni_d (temp_d[30]), - qm_d (temp_d[31]), - bm_d (temp_d[32]), - latent_heat_vapor_d (temp_d[33]), - latent_heat_sublim_d(temp_d[34]), - latent_heat_fusion_d(temp_d[35]), - qc_incld_d (temp_d[36]), - qr_incld_d (temp_d[37]), - qi_incld_d (temp_d[38]), - qm_incld_d (temp_d[39]), - nc_incld_d (temp_d[40]), - nr_incld_d (temp_d[41]), - ni_incld_d (temp_d[42]), - bm_incld_d (temp_d[43]), - mu_c_d (temp_d[44]), - nu_d (temp_d[45]), - lamc_d (temp_d[46]), - cdist_d (temp_d[47]), - cdist1_d (temp_d[48]), - cdistr_d (temp_d[49]), - mu_r_d (temp_d[50]), - lamr_d (temp_d[51]), - logn0r_d (temp_d[52]), - qv2qi_depos_tend_d (temp_d[53]), - precip_total_tend_d (temp_d[54]), - nevapr_d (temp_d[55]), - qr_evap_tend_d (temp_d[56]), - vap_liq_exchange_d (temp_d[57]), - vap_ice_exchange_d (temp_d[58]), - liq_ice_exchange_d (temp_d[59]), - pratot_d (temp_d[60]), - prctot_d (temp_d[61]), - qv_prev_d (temp_d[62]), - t_prev_d (temp_d[63]); + pres_d (temp_d[current_index++]), + dpres_d (temp_d[current_index++]), + dz_d (temp_d[current_index++]), + nc_nuceat_tend_d (temp_d[current_index++]), + inv_exner_d (temp_d[current_index++]), + exner_d (temp_d[current_index++]), + inv_cld_frac_l_d (temp_d[current_index++]), + inv_cld_frac_i_d (temp_d[current_index++]), + inv_cld_frac_r_d (temp_d[current_index++]), + ni_activated_d (temp_d[current_index++]), + inv_qc_relvar_d (temp_d[current_index++]), + cld_frac_i_d (temp_d[current_index++]), + cld_frac_l_d (temp_d[current_index++]), + cld_frac_r_d (temp_d[current_index++]), + t_d (temp_d[current_index++]), + rho_d (temp_d[current_index++]), + inv_rho_d (temp_d[current_index++]), + qv_sat_l_d (temp_d[current_index++]), + qv_sat_i_d (temp_d[current_index++]), + qv_supersat_i_d (temp_d[current_index++]), + rhofacr_d (temp_d[current_index++]), + rhofaci_d (temp_d[current_index++]), + acn_d (temp_d[current_index++]), + qv_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + qc_incld_d (temp_d[current_index++]), + qr_incld_d (temp_d[current_index++]), + qi_incld_d (temp_d[current_index++]), + qm_incld_d (temp_d[current_index++]), + nc_incld_d (temp_d[current_index++]), + nr_incld_d (temp_d[current_index++]), + ni_incld_d (temp_d[current_index++]), + bm_incld_d (temp_d[current_index++]), + mu_c_d (temp_d[current_index++]), + nu_d (temp_d[current_index++]), + lamc_d (temp_d[current_index++]), + cdist_d (temp_d[current_index++]), + cdist1_d (temp_d[current_index++]), + cdistr_d (temp_d[current_index++]), + mu_r_d (temp_d[current_index++]), + lamr_d (temp_d[current_index++]), + logn0r_d (temp_d[current_index++]), + qv2qi_depos_tend_d (temp_d[current_index++]), + precip_total_tend_d (temp_d[current_index++]), + nevapr_d (temp_d[current_index++]), + qr_evap_tend_d (temp_d[current_index++]), + vap_liq_exchange_d (temp_d[current_index++]), + vap_ice_exchange_d (temp_d[current_index++]), + liq_ice_exchange_d (temp_d[current_index++]), + pratot_d (temp_d[current_index++]), + prctot_d (temp_d[current_index++]), + qv_prev_d (temp_d[current_index++]), + t_prev_d (temp_d[current_index++]); // Call core function from kernel const auto dnu = P3GlobalForFortran::dnu(); @@ -1780,7 +1744,7 @@ void p3_main_part2_f( inv_cld_frac_i_d, inv_cld_frac_r_d, ni_activated_d, inv_qc_relvar_d, cld_frac_i_d, cld_frac_l_d, cld_frac_r_d, qv_prev_d, t_prev_d, t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, - latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, qc_incld_d, qr_incld_d, qi_incld_d, + qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, @@ -1791,7 +1755,7 @@ void p3_main_part2_f( std::vector inout_views = { t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, - latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, + qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, vap_ice_exchange_d, @@ -1800,7 +1764,7 @@ void p3_main_part2_f( ekat::device_to_host({ T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, - qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, + qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, @@ -1818,7 +1782,7 @@ void p3_main_part3_f( Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, - Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* mu_c, Real* nu, Real* lamc, + Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr) @@ -1844,50 +1808,49 @@ void p3_main_part3_f( const Real max_total_ni = 740.0e3; // Hard-code this value for F90 comparison // Set up views - std::vector temp_d(P3MainPart3Data::NUM_ARRAYS); + std::vector temp_d(P3MainPart3Data::NUM_ARRAYS-2); ekat::host_to_device({ inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, rho, inv_rho, rhofaci, qv, th_atm, qc, - nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, + nc, qr, nr, qi, ni, qm, bm, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr}, nk, temp_d); + int current_index = 0; view_1d - inv_exner_d (temp_d[0]), - cld_frac_l_d (temp_d[1]), - cld_frac_r_d (temp_d[2]), - cld_frac_i_d (temp_d[3]), - rho_d (temp_d[4]), - inv_rho_d (temp_d[5]), - rhofaci_d (temp_d[6]), - qv_d (temp_d[7]), - th_atm_d (temp_d[8]), - qc_d (temp_d[9]), - nc_d (temp_d[10]), - qr_d (temp_d[11]), - nr_d (temp_d[12]), - qi_d (temp_d[13]), - ni_d (temp_d[14]), - qm_d (temp_d[15]), - bm_d (temp_d[16]), - latent_heat_vapor_d (temp_d[17]), - latent_heat_sublim_d (temp_d[18]), - mu_c_d (temp_d[19]), - nu_d (temp_d[20]), - lamc_d (temp_d[21]), - mu_r_d (temp_d[22]), - lamr_d (temp_d[23]), - vap_liq_exchange_d (temp_d[24]), - ze_rain_d (temp_d[25]), - ze_ice_d (temp_d[26]), - diag_vm_qi_d (temp_d[27]), - diag_eff_radius_qi_d (temp_d[28]), - diag_diam_qi_d (temp_d[29]), - rho_qi_d (temp_d[30]), - diag_equiv_reflectivity_d (temp_d[31]), - diag_eff_radius_qc_d (temp_d[32]), - diag_eff_radius_qr_d (temp_d[33]); + inv_exner_d (temp_d[current_index++]), + cld_frac_l_d (temp_d[current_index++]), + cld_frac_r_d (temp_d[current_index++]), + cld_frac_i_d (temp_d[current_index++]), + rho_d (temp_d[current_index++]), + inv_rho_d (temp_d[current_index++]), + rhofaci_d (temp_d[current_index++]), + qv_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + mu_c_d (temp_d[current_index++]), + nu_d (temp_d[current_index++]), + lamc_d (temp_d[current_index++]), + mu_r_d (temp_d[current_index++]), + lamr_d (temp_d[current_index++]), + vap_liq_exchange_d (temp_d[current_index++]), + ze_rain_d (temp_d[current_index++]), + ze_ice_d (temp_d[current_index++]), + diag_vm_qi_d (temp_d[current_index++]), + diag_eff_radius_qi_d (temp_d[current_index++]), + diag_diam_qi_d (temp_d[current_index++]), + rho_qi_d (temp_d[current_index++]), + diag_equiv_reflectivity_d (temp_d[current_index++]), + diag_eff_radius_qc_d (temp_d[current_index++]), + diag_eff_radius_qr_d (temp_d[current_index++]); // Call core function from kernel const auto dnu = P3GlobalForFortran::dnu(); @@ -1898,8 +1861,8 @@ void p3_main_part3_f( P3F::p3_main_part3(team, nk_pack, max_total_ni, dnu, ice_table_vals, inv_exner_d, cld_frac_l_d, cld_frac_r_d, cld_frac_i_d, rho_d, inv_rho_d, rhofaci_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, - qi_d, ni_d, qm_d, bm_d, latent_heat_vapor_d, - latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, + qi_d, ni_d, qm_d, bm_d, + mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d, physics::P3_Constants()); @@ -1908,14 +1871,14 @@ void p3_main_part3_f( // Sync back to host std::vector inout_views = { rho_d, inv_rho_d, rhofaci_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, - ni_d, qm_d, bm_d, latent_heat_vapor_d, latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, + ni_d, qm_d, bm_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d }; ekat::device_to_host({ rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, - latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, + mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr }, diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 926c9ab47017..d1c29d3de0de 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -908,11 +908,9 @@ void rain_sedimentation_f( void homogeneous_freezing_f( Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, + Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); -void get_latent_heat_f(Int its, Int ite, Int kts, Int kte, Real* v, Real* s, Real* f); - void check_values_f(Real* Qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); @@ -921,7 +919,7 @@ void p3_main_part1_f( bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, + Real* inv_cld_frac_r, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, @@ -931,7 +929,7 @@ void p3_main_part2_f( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, + Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present); @@ -939,7 +937,7 @@ void p3_main_part2_f( void p3_main_part3_f( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, - Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, + Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 912c7cedac66..217c2945e489 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -26,7 +26,6 @@ set(P3_TESTS_SRCS p3_ice_melting_unit_tests.cpp p3_evaporate_rain_unit_tests.cpp p3_ice_cldliq_wet_growth_unit_tests.cpp - p3_get_latent_heat_unit_tests.cpp p3_subgrid_variance_scaling_unit_tests.cpp p3_check_values_unit_tests.cpp p3_incloud_mixingratios_unit_tests.cpp diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index 24cc3a848188..f8398135a19e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -82,7 +82,6 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Spack epsi_tot(1./60.); Spack t(287); Spack t_prev(285); - Spack latent_heat_sublim(3.34e5); Spack dqsdt(1e-3); Scalar dt=60; Spack qrtend; @@ -92,7 +91,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip constexpr Scalar QSMALL = C::QSMALL; Functions::evaporate_rain(Spack(QSMALL/2),qc_incld,nr_incld,qi_incld, //qr_incld->QSMALL/2 cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -101,7 +100,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Spack qr_tiny=Spack(5e-13); Functions::evaporate_rain(qr_tiny,qc_incld,nr_incld,qi_incld, //qr_incld->_tiny cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0] - qr_tiny[0]/dt *(cld_frac_r[0]-cld_frac_l[0])/cld_frac_r[0])<1e-8 ); @@ -111,7 +110,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip //if no rainy areas outside cloud, don't evap Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_r,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, //cld_frac_l->_r - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -120,7 +119,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, //set qv->qv_sat_l*2 in next line to ensure supersaturated. cld_frac_l,cld_frac_r,qv_sat_l*2,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -129,7 +128,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, //qv -> qv*0.1 to encourage lots of rain evap cld_frac_l,cld_frac_r,qv*0.1,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( qrtend[0] <= qr_incld[0]/dt); REQUIRE( nrtend[0] <= nr_incld[0]/dt); //keep end-of-step nr positive. Should always be true. @@ -137,6 +136,8 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip }; //end run_property static void run_bfb(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following //This subroutine has 20 args, only 18 are supplied here for invoking it as last 2 are intent-outs @@ -152,23 +153,23 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip //rows 9-16: random junk but ensured cld_frac_r>cld_frac_l and subsaturated. EvapRainData espd[max_pack_size] = { //qr_incld, qc_incld, nr_incld, qi_incld, cld_frac_l, cld_frac_r, qv, qv_prev, qv_sat_l, qv_sat_i, ab, abi, epsr, epsi_tot, t, t_prev, lat_ht_sublim, dqsdt, dt - {4.634940e-03,1.215335e-03,6.073270e+07,3.594486e-04,6.134229e-01,9.134229e-01,2.747871e-03,1.911238e-03,5.913313e-03,1.057645e-03,1.782748e+00,1.571392e+00,3.868229e+02,2.248689e+02,3.101180e+02,1.395063e+02,3.335413e+05,5.494606e-03,6.000000e+02}, - {6.175320e-13,4.432407e-03,8.029967e+07,1.905151e-03,2.190099e-01,7.031070e-01,4.172977e-05,7.315360e-03,7.280063e-03,1.378543e-03,1.461443e+00,1.507382e+00,8.452377e+02,1.971876e+02,2.389249e+02,1.497752e+02,3.578466e+05,5.107905e-03,6.000000e+02}, - {4.519798e-03,7.348916e-03,7.420725e+07,2.220971e-03,1.882608e-01,2.934182e-01,4.957590e-03,2.550256e-03,3.136926e-03,4.498115e-03,1.433526e+00,1.207516e+00,9.716844e+02,5.602546e+01,1.389465e+02,1.075863e+02,3.570404e+05,6.771428e-03,6.000000e+02}, - {7.169182e-03,6.657331e-03,9.807967e+07,7.981196e-03,2.914473e-01,6.375719e-01,2.420032e-03,1.223012e-03,7.685516e-03,5.207024e-03,1.644865e+00,1.433872e+00,3.825069e+02,6.550300e+02,1.833466e+02,1.741918e+02,3.295800e+05,3.792982e-03,6.000000e+02}, - {1.103118e-03,9.158125e-03,3.136196e+07,4.286154e-03,2.699078e-01,4.668103e-01,9.645460e-03,6.379119e-03,8.283285e-03,3.342400e-03,1.546698e+00,1.417916e+00,9.289270e+02,9.844129e+02,2.543202e+02,1.932996e+02,3.327786e+05,2.693119e-03,6.000000e+02}, - {4.308000e-03,8.168535e-03,7.439969e+07,5.131497e-03,6.851225e-01,3.298025e-01,4.331812e-03,2.814373e-03,3.592807e-03,1.527499e-03,1.856943e+00,1.003269e+00,9.165690e+02,9.379921e+02,2.163204e+02,3.165814e+02,3.874371e+05,6.801393e-03,6.000000e+02}, - {0.000000e-00,3.318968e-03,4.664041e+07,8.737282e-03,2.585907e-01,6.297295e-02,8.747418e-03,2.710437e-03,2.164895e-03,9.455725e-03,1.241506e+00,1.561393e+00,2.492674e+02,6.546182e+02,2.228772e+02,2.147968e+02,3.590381e+05,5.903261e-03,6.000000e+02}, - {7.677170e-03,6.069057e-05,6.404241e+07,3.094233e-03,3.755403e-01,5.026876e-01,4.723817e-03,1.204228e-03,6.156526e-03,8.194797e-03,1.361509e+00,1.772751e+00,6.420537e+01,4.043364e+02,2.833110e+02,3.314521e+02,3.427004e+05,2.996696e-03,6.000000e+02}, - - {9.999294e-03,3.138400e-03,2.355097e+07,9.897893e-03,7.667177e-01,9.739270e-01,4.221430e-03,3.570130e-03,8.370033e-03,9.527208e-03,1.597218e+00,1.111438e+00,7.832357e+02,8.364566e+02,2.854867e+02,2.340771e+02,3.198709e+05,7.235757e-03,6.000000e+02}, - {8.841793e-03,3.530456e-03,9.618284e+07,9.311658e-03,3.458590e-01,6.978258e-01,1.279864e-03,4.652008e-03,1.869728e-03,8.931663e-03,1.712564e+00,1.223882e+00,9.692403e+02,2.358558e+02,3.204043e+02,1.827677e+02,3.220502e+05,7.646405e-03,6.000000e+02}, - {1.425612e-03,6.653411e-04,2.843806e+07,1.922560e-03,9.100262e-01,0.996264e-01,8.973183e-04,9.857420e-03,6.221419e-03,8.133433e-03,1.815337e+00,1.885506e+00,5.508742e+02,1.612139e+02,2.798523e+02,2.631136e+02,3.045141e+05,4.148666e-03,6.000000e+02}, - {4.125177e-04,4.056163e-03,2.716439e+07,6.484214e-03,1.658752e-01,2.859102e-01,5.724081e-03,6.282997e-03,7.313187e-03,6.049825e-03,1.140910e+00,1.145941e+00,7.490652e+02,5.011633e+02,1.986541e+02,2.745566e+02,3.371001e+05,6.784784e-03,6.000000e+02}, - {5.010628e-03,2.863789e-04,8.953841e+07,3.953058e-03,1.135952e-01,9.718675e-01,1.846157e-03,5.743094e-03,2.842649e-03,8.155366e-03,1.227867e+00,1.894249e+00,1.161776e+02,3.578576e+02,1.240083e+02,1.639791e+02,3.167181e+05,4.497257e-03,6.000000e+02}, - {9.487866e-03,6.584660e-03,6.149682e+06,9.413342e-03,4.757261e-01,6.503885e-01,1.078922e-03,3.489665e-03,3.059596e-03,9.285703e-03,1.192620e+00,1.967205e+00,5.085628e+02,3.741816e+01,1.196252e+02,2.904002e+02,3.637035e+05,2.566077e-03,6.000000e+02}, - {3.241928e-03,7.024929e-03,2.212493e+07,8.600485e-03,3.963690e-01,4.834201e-01,3.736511e-03,5.724475e-03,4.790239e-03,2.766218e-03,1.151150e+00,1.150516e+00,2.089426e+02,8.666450e+02,1.898220e+02,2.862496e+02,3.056143e+05,7.039800e-03,6.000000e+02}, - {4.617594e-03,3.157739e-03,5.569465e+07,8.221076e-03,7.918279e-01,9.995014e-01,1.338309e-04,1.319707e-03,2.896082e-03,4.359171e-03,1.007827e+00,1.812954e+00,5.332209e+02,2.973599e+02,3.271466e+02,2.622351e+02,3.821569e+05,1.407429e-03,6.000000e+02} + {4.634940e-03,1.215335e-03,6.073270e+07,3.594486e-04,6.134229e-01,9.134229e-01,2.747871e-03,1.911238e-03,5.913313e-03,1.057645e-03,1.782748e+00,1.571392e+00,3.868229e+02,2.248689e+02,3.101180e+02,1.395063e+02,latvap+latice,5.494606e-03,6.000000e+02}, + {6.175320e-13,4.432407e-03,8.029967e+07,1.905151e-03,2.190099e-01,7.031070e-01,4.172977e-05,7.315360e-03,7.280063e-03,1.378543e-03,1.461443e+00,1.507382e+00,8.452377e+02,1.971876e+02,2.389249e+02,1.497752e+02,latvap+latice,5.107905e-03,6.000000e+02}, + {4.519798e-03,7.348916e-03,7.420725e+07,2.220971e-03,1.882608e-01,2.934182e-01,4.957590e-03,2.550256e-03,3.136926e-03,4.498115e-03,1.433526e+00,1.207516e+00,9.716844e+02,5.602546e+01,1.389465e+02,1.075863e+02,latvap+latice,6.771428e-03,6.000000e+02}, + {7.169182e-03,6.657331e-03,9.807967e+07,7.981196e-03,2.914473e-01,6.375719e-01,2.420032e-03,1.223012e-03,7.685516e-03,5.207024e-03,1.644865e+00,1.433872e+00,3.825069e+02,6.550300e+02,1.833466e+02,1.741918e+02,latvap+latice,3.792982e-03,6.000000e+02}, + {1.103118e-03,9.158125e-03,3.136196e+07,4.286154e-03,2.699078e-01,4.668103e-01,9.645460e-03,6.379119e-03,8.283285e-03,3.342400e-03,1.546698e+00,1.417916e+00,9.289270e+02,9.844129e+02,2.543202e+02,1.932996e+02,latvap+latice,2.693119e-03,6.000000e+02}, + {4.308000e-03,8.168535e-03,7.439969e+07,5.131497e-03,6.851225e-01,3.298025e-01,4.331812e-03,2.814373e-03,3.592807e-03,1.527499e-03,1.856943e+00,1.003269e+00,9.165690e+02,9.379921e+02,2.163204e+02,3.165814e+02,latvap+latice,6.801393e-03,6.000000e+02}, + {0.000000e-00,3.318968e-03,4.664041e+07,8.737282e-03,2.585907e-01,6.297295e-02,8.747418e-03,2.710437e-03,2.164895e-03,9.455725e-03,1.241506e+00,1.561393e+00,2.492674e+02,6.546182e+02,2.228772e+02,2.147968e+02,latvap+latice,5.903261e-03,6.000000e+02}, + {7.677170e-03,6.069057e-05,6.404241e+07,3.094233e-03,3.755403e-01,5.026876e-01,4.723817e-03,1.204228e-03,6.156526e-03,8.194797e-03,1.361509e+00,1.772751e+00,6.420537e+01,4.043364e+02,2.833110e+02,3.314521e+02,latvap+latice,2.996696e-03,6.000000e+02}, + + {9.999294e-03,3.138400e-03,2.355097e+07,9.897893e-03,7.667177e-01,9.739270e-01,4.221430e-03,3.570130e-03,8.370033e-03,9.527208e-03,1.597218e+00,1.111438e+00,7.832357e+02,8.364566e+02,2.854867e+02,2.340771e+02,latvap+latice,7.235757e-03,6.000000e+02}, + {8.841793e-03,3.530456e-03,9.618284e+07,9.311658e-03,3.458590e-01,6.978258e-01,1.279864e-03,4.652008e-03,1.869728e-03,8.931663e-03,1.712564e+00,1.223882e+00,9.692403e+02,2.358558e+02,3.204043e+02,1.827677e+02,latvap+latice,7.646405e-03,6.000000e+02}, + {1.425612e-03,6.653411e-04,2.843806e+07,1.922560e-03,9.100262e-01,0.996264e-01,8.973183e-04,9.857420e-03,6.221419e-03,8.133433e-03,1.815337e+00,1.885506e+00,5.508742e+02,1.612139e+02,2.798523e+02,2.631136e+02,latvap+latice,4.148666e-03,6.000000e+02}, + {4.125177e-04,4.056163e-03,2.716439e+07,6.484214e-03,1.658752e-01,2.859102e-01,5.724081e-03,6.282997e-03,7.313187e-03,6.049825e-03,1.140910e+00,1.145941e+00,7.490652e+02,5.011633e+02,1.986541e+02,2.745566e+02,latvap+latice,6.784784e-03,6.000000e+02}, + {5.010628e-03,2.863789e-04,8.953841e+07,3.953058e-03,1.135952e-01,9.718675e-01,1.846157e-03,5.743094e-03,2.842649e-03,8.155366e-03,1.227867e+00,1.894249e+00,1.161776e+02,3.578576e+02,1.240083e+02,1.639791e+02,latvap+latice,4.497257e-03,6.000000e+02}, + {9.487866e-03,6.584660e-03,6.149682e+06,9.413342e-03,4.757261e-01,6.503885e-01,1.078922e-03,3.489665e-03,3.059596e-03,9.285703e-03,1.192620e+00,1.967205e+00,5.085628e+02,3.741816e+01,1.196252e+02,2.904002e+02,latvap+latice,2.566077e-03,6.000000e+02}, + {3.241928e-03,7.024929e-03,2.212493e+07,8.600485e-03,3.963690e-01,4.834201e-01,3.736511e-03,5.724475e-03,4.790239e-03,2.766218e-03,1.151150e+00,1.150516e+00,2.089426e+02,8.666450e+02,1.898220e+02,2.862496e+02,latvap+latice,7.039800e-03,6.000000e+02}, + {4.617594e-03,3.157739e-03,5.569465e+07,8.221076e-03,7.918279e-01,9.995014e-01,1.338309e-04,1.319707e-03,2.896082e-03,4.359171e-03,1.007827e+00,1.812954e+00,5.332209e+02,2.973599e+02,3.271466e+02,2.622351e+02,latvap+latice,1.407429e-03,6.000000e+02} }; // Sync to device @@ -191,7 +192,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip // Init pack inputs Spack qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt; + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt; Scalar dt; @@ -215,7 +216,6 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip epsi_tot[s] = espd_device(vs).epsi_tot; t[s] = espd_device(vs).t; t_prev[s] = espd_device(vs).t_prev; - latent_heat_sublim[s]=espd_device(vs).latent_heat_sublim; dqsdt[s]=espd_device(vs).dqsdt; dt=espd_device(vs).dt; //qr2qv_evap_tend[s] = espd_device(vs).qr2qv_evap_tend; //PMC shouldn't have to init output vars. @@ -224,7 +224,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qr2qv_evap_tend,nr_evap_tend); // Copy results back into views diff --git a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp index 3ce02ced4597..5d7b4de4ea7e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp @@ -24,12 +24,12 @@ struct UnitWrap::UnitTest::TestLatentHeat { static void run_latent_heat_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + LatentHeatData latent_fortran[] = { // its, ite, kts, kte LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), }; static constexpr Int num_runs = sizeof(latent_fortran) / sizeof(LatentHeatData); @@ -45,16 +45,11 @@ struct UnitWrap::UnitTest::TestLatentHeat { LatentHeatData& h = latent_fortran[i]; get_latent_heat(h); - LatentHeatData& d = latent_cxx[i]; - get_latent_heat_f(d.its, d.ite, d.kts, d.kte, d.v, d.s, d.f); - if (SCREAM_BFB_TESTING) { - REQUIRE(h.total(h.v) == d.total(d.v)); - for (Int j = 0; j < h.total(h.v); ++j) { - REQUIRE(d.v[j] == h.v[j]); - REQUIRE(d.s[j] == h.s[j]); - REQUIRE(d.f[j] == h.f[j]); + REQUIRE(h.v[j] == latvap); + REQUIRE(h.s[j] == (latvap+latice)); + REQUIRE(h.f[j] == latice); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index 78f7ab986c66..caf6638d5cd5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -25,27 +25,30 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { { using KTH = KokkosTypes; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + IceWetGrowthData self[max_pack_size] = { // rho,temp,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,kap,mu,sc,qv,qc_incld,qi_incld,ni_incld,qr_incld,log_wetgrowth,qr2qi_collect_tend,qc2qi_collect_tend,qc_growth_rate,nr_ice_shed_tend,qc2qr_ice_shed_tend - {4.056000E-03, 1.023000E+02, 1.201000E+02, 9.002000E-04, 8.215000E-04, 8.852000E-01, 0.174000E+00, 1.221000E-14, 5.100000E-03, 9.558000E-04, 1.213000E-03, 9.653000E-04, 1.023000E-01, 4.098000E-02, 2.098000E-02, 9.952000E+03, 1.023000E-05, false, 1.241456E-04, 9.021345E-02, 1.043000E-01, 1.921000E-02, 0.242000E-02}, - {6.852000E-02, 1.120000E+02, 2.450000E+02, 9.321000E-04, 9.124000E-04, 8.852000E-01, 0.374000E+00, 1.221000E-13, 4.100000E-03, 9.558000E-04, 2.560000E-03, 1.764000E-03, 2.346000E-01, 5.632000E-02, 3.024000E-02, 9.952000E+03, 2.093000E-05, false, 2.341678E-04, 1.092432E-02, 2.903000E-01, 2.125000E-02, 0.342000E-02}, - {8.852000E-02, 1.210000E+02, 3.420000E+02, 9.623000E-04, 9.432000E-04, 8.900000E-01, 0.123000E+00, 1.221000E-12, 3.100000E-03, 9.558000E-04, 3.211000E-03, 3.421000E-03, 3.421000E-01, 6.542000E-02, 4.567000E-02, 9.952000E+03, 3.091000E-05, false, 3.215234E-04, 2.098987E-02, 3.450000E-01, 3.490000E-02, 0.932000E-02}, - {1.902000E-01, 1.326000E+02, 4.321000E+02, 9.982000E-04, 9.623000E-04, 9.900000E-01, 0.123000E+00, 1.221000E-11, 2.100000E-03, 9.558000E-04, 4.121000E-03, 4.569000E-03, 4.673000E-01, 7.902000E-02, 5.321000E-02, 9.952000E+03, 4.521000E-05, false, 4.675567E-04, 3.214982E-02, 4.290000E-01, 4.590000E-02, 1.025000E-01}, - - {2.201000E-01, 1.456000E+02, 5.670000E+02, 1.234000E-03, 9.723000E-04, 0.100000E+01, 0.174000E+00, 1.221000E-10, 1.100000E-03, 2.550008E-05, 4.980000E-03, 5.621000E-03, 5.420000E-01, 8.021000E-02, 6.902000E-02, 9.952000E+04, 5.678000E-05, false, 5.389236E-04, 4.125969E-02, 5.098000E-01, 5.921000E-02, 2.031000E-01}, - {3.502000E-01, 1.780009E+02, 6.832000E+02, 1.562000E-03, 1.024000E-03, 0.100000E+01, 0.374000E+00, 1.221000E-09, 8.100000E-04, 2.558000E-05, 5.643000E-03, 7.367000E-03, 6.782000E-01, 9.253000E-02, 8.045000E-02, 9.952000E+04, 6.902000E-05, false, 6.432654E-04, 5.389457E-02, 6.723000E-01, 6.093000E-02, 4.098000E-01}, - {4.852000E-01, 2.100009E+02, 7.090000E+02, 2.101000E-03, 1.235000E-03, 0.100000E+01, 0.123000E+00, 1.221000E-08, 4.100000E-04, 2.558000E-05, 7.892000E-03, 9.087000E-03, 8.213000E-01, 1.256000E-01, 9.134000E-02, 9.952000E+04, 8.367000E-05, false, 7.210983E-04, 6.476985E-02, 8.902000E-01, 8.345000E-02, 8.023000E-01}, - {5.852000E-01, 2.310000E+02, 9.215000E+02, 2.312000E-03, 1.456000E-03, 0.100000E+01, 0.123000E+00, 1.221000E-07, 2.100000E-04, 2.558000E-05, 9.321000E-03, 1.245000E-02, 1.067000E-00, 2.347000E-01, 1.092000E-01, 9.952000E+04, 9.098000E-05, false, 8.543367E-04, 8.213186E-02, 9.021000E-01, 9.321000E-02, 9.098000E-01}, - - {6.852000E-01, 2.563000E+02, 1.089000E+03, 3.601000E-03, 1.864000E-03, 0.950000E+00, 0.150000E+00, 1.221000E-06, 9.952000E-05, 4.596000E-05, 1.453000E-02, 2.543000E-02, 2.345000E-00, 3.578000E-01, 2.873000E-01, 1.734000E+04, 1.023000E-04, false, 9.021215E-04, 9.023367E-02, 1.023000E-00, 1.056000E-01, 1.256000E-00}, - {7.852000E-01, 2.789000E+02, 3.754000E+03, 3.891000E-03, 2.093000E-03, 0.950000E+00, 0.374000E+00, 1.221000E-05, 4.952000E-05, 4.596000E-05, 2.789000E-02, 4.367000E-02, 3.890000E-00, 4.980000E-01, 3.468000E-01, 1.734000E+04, 2.146000E-04, false, 1.043468E-05, 1.094854E-02, 2.012000E-00, 2.893000E-01, 2.903000E-00}, - {8.852000E-01, 3.123000E+02, 8.902000E+03, 4.872000E-03, 2.345000E-03, 0.950000E+00, 0.123000E+00, 1.221000E-04, 1.952000E-05, 4.596000E-05, 4.256000E-02, 6.324000E-02, 4.120000E-00, 6.321000E-01, 4.890000E-01, 1.734000E+04, 4.321000E-04, false, 2.341763E-05, 2.126247E-03, 3.120000E-00, 3.456000E-01, 3.912000E-00}, - {9.852000E-01, 4.981000E+02, 1.092000E+04, 5.210000E-03, 3.210000E-03, 0.950000E+00, 0.123000E+00, 1.221000E-03, 9.952000E-06, 4.596000E-05, 6.821000E-02, 8.789000E-02, 5.320000E-00, 7.982000E-01, 6.921000E-01, 1.734000E+04, 5.821000E-04, false, 3.901479E-05, 3.874763E-03, 5.902000E-00, 5.092000E-01, 4.821000E-00}, - - {1.002000E+01, 1.234000E+03, 2.125000E+04, 6.012000E-03, 5.902000E-03, 1.069000E+00, 0.174000E+00, 1.221000E-02, 6.952000E-06, 6.596000E-05, 8.472000E-02, 1.543000E-01, 6.012000E-00, 8.902000E-01, 9.210000E-01, 1.734000E+04, 6.921000E-04, false, 4.521923E-05, 4.592698E-03, 6.091000E-00, 6.743000E-01, 5.602000E-00}, - {1.152000E+01, 2.120000E+03, 4.568000E+04, 6.342000E-03, 9.210000E-03, 1.069000E+00, 0.374000E+00, 1.221000E-02, 3.952000E-06, 6.596000E-05, 1.098000E-01, 3.456000E-01, 7.241000E-00, 9.102000E-01, 1.002000E-00, 1.734000E+04, 7.901000E-04, false, 5.236542E-05, 5.678873E-03, 7.231000E-00, 8.321000E-01, 6.092000E-00}, - {1.252000E+01, 3.145000E+03, 8.213000E+04, 9.290000E-03, 1.034000E-02, 1.069000E+00, 0.123000E+00, 1.221000E-02, 1.952000E-06, 6.596000E-05, 2.340006E-01, 5.632000E-01, 8.452000E-00, 1.003000E-01, 2.145000E-00, 1.734000E+04, 9.212000E-04, false, 6.732276E-05, 7.321873E-03, 8.234000E-00, 9.023000E-01, 7.201000E-00}, - {1.352000E+01, 4.742000E+03, 1.014000E+05, 1.234000E-02, 1.456000E-02, 1.069000E+00, 0.123000E+00, 1.221000E-02, 9.952000E-07, 6.596000E-05, 4.123000E-01, 6.128000E-01, 9.076000E-00, 2.831000E-01, 3.902000E-00, 1.734000E+04, 1.023000E-03, false, 7.902887E-05, 9.032908E-03, 9.021000E-00, 1.092000E-01, 8.096000E-00} + {4.056000E-03, 1.023000E+02, 1.201000E+02, 9.002000E-04, 8.215000E-04, 8.852000E-01, latvap, latice, 5.100000E-03, 9.558000E-04, 1.213000E-03, 9.653000E-04, 1.023000E-01, 4.098000E-02, 2.098000E-02, 9.952000E+03, 1.023000E-05, false, 1.241456E-04, 9.021345E-02, 1.043000E-01, 1.921000E-02, 0.242000E-02}, + {6.852000E-02, 1.120000E+02, 2.450000E+02, 9.321000E-04, 9.124000E-04, 8.852000E-01, latvap, latice, 4.100000E-03, 9.558000E-04, 2.560000E-03, 1.764000E-03, 2.346000E-01, 5.632000E-02, 3.024000E-02, 9.952000E+03, 2.093000E-05, false, 2.341678E-04, 1.092432E-02, 2.903000E-01, 2.125000E-02, 0.342000E-02}, + {8.852000E-02, 1.210000E+02, 3.420000E+02, 9.623000E-04, 9.432000E-04, 8.900000E-01, latvap, latice, 3.100000E-03, 9.558000E-04, 3.211000E-03, 3.421000E-03, 3.421000E-01, 6.542000E-02, 4.567000E-02, 9.952000E+03, 3.091000E-05, false, 3.215234E-04, 2.098987E-02, 3.450000E-01, 3.490000E-02, 0.932000E-02}, + {1.902000E-01, 1.326000E+02, 4.321000E+02, 9.982000E-04, 9.623000E-04, 9.900000E-01, latvap, latice, 2.100000E-03, 9.558000E-04, 4.121000E-03, 4.569000E-03, 4.673000E-01, 7.902000E-02, 5.321000E-02, 9.952000E+03, 4.521000E-05, false, 4.675567E-04, 3.214982E-02, 4.290000E-01, 4.590000E-02, 1.025000E-01}, + + {2.201000E-01, 1.456000E+02, 5.670000E+02, 1.234000E-03, 9.723000E-04, 0.100000E+01, latvap, latice, 1.100000E-03, 2.550008E-05, 4.980000E-03, 5.621000E-03, 5.420000E-01, 8.021000E-02, 6.902000E-02, 9.952000E+04, 5.678000E-05, false, 5.389236E-04, 4.125969E-02, 5.098000E-01, 5.921000E-02, 2.031000E-01}, + {3.502000E-01, 1.780009E+02, 6.832000E+02, 1.562000E-03, 1.024000E-03, 0.100000E+01, latvap, latice, 8.100000E-04, 2.558000E-05, 5.643000E-03, 7.367000E-03, 6.782000E-01, 9.253000E-02, 8.045000E-02, 9.952000E+04, 6.902000E-05, false, 6.432654E-04, 5.389457E-02, 6.723000E-01, 6.093000E-02, 4.098000E-01}, + {4.852000E-01, 2.100009E+02, 7.090000E+02, 2.101000E-03, 1.235000E-03, 0.100000E+01, latvap, latice, 4.100000E-04, 2.558000E-05, 7.892000E-03, 9.087000E-03, 8.213000E-01, 1.256000E-01, 9.134000E-02, 9.952000E+04, 8.367000E-05, false, 7.210983E-04, 6.476985E-02, 8.902000E-01, 8.345000E-02, 8.023000E-01}, + {5.852000E-01, 2.310000E+02, 9.215000E+02, 2.312000E-03, 1.456000E-03, 0.100000E+01, latvap, latice, 2.100000E-04, 2.558000E-05, 9.321000E-03, 1.245000E-02, 1.067000E-00, 2.347000E-01, 1.092000E-01, 9.952000E+04, 9.098000E-05, false, 8.543367E-04, 8.213186E-02, 9.021000E-01, 9.321000E-02, 9.098000E-01}, + + {6.852000E-01, 2.563000E+02, 1.089000E+03, 3.601000E-03, 1.864000E-03, 0.950000E+00, latvap, latice, 9.952000E-05, 4.596000E-05, 1.453000E-02, 2.543000E-02, 2.345000E-00, 3.578000E-01, 2.873000E-01, 1.734000E+04, 1.023000E-04, false, 9.021215E-04, 9.023367E-02, 1.023000E-00, 1.056000E-01, 1.256000E-00}, + {7.852000E-01, 2.789000E+02, 3.754000E+03, 3.891000E-03, 2.093000E-03, 0.950000E+00, latvap, latice, 4.952000E-05, 4.596000E-05, 2.789000E-02, 4.367000E-02, 3.890000E-00, 4.980000E-01, 3.468000E-01, 1.734000E+04, 2.146000E-04, false, 1.043468E-05, 1.094854E-02, 2.012000E-00, 2.893000E-01, 2.903000E-00}, + {8.852000E-01, 3.123000E+02, 8.902000E+03, 4.872000E-03, 2.345000E-03, 0.950000E+00, latvap, latice, 1.952000E-05, 4.596000E-05, 4.256000E-02, 6.324000E-02, 4.120000E-00, 6.321000E-01, 4.890000E-01, 1.734000E+04, 4.321000E-04, false, 2.341763E-05, 2.126247E-03, 3.120000E-00, 3.456000E-01, 3.912000E-00}, + {9.852000E-01, 4.981000E+02, 1.092000E+04, 5.210000E-03, 3.210000E-03, 0.950000E+00, latvap, latice, 9.952000E-06, 4.596000E-05, 6.821000E-02, 8.789000E-02, 5.320000E-00, 7.982000E-01, 6.921000E-01, 1.734000E+04, 5.821000E-04, false, 3.901479E-05, 3.874763E-03, 5.902000E-00, 5.092000E-01, 4.821000E-00}, + + {1.002000E+01, 1.234000E+03, 2.125000E+04, 6.012000E-03, 5.902000E-03, 1.069000E+00, latvap, latice, 6.952000E-06, 6.596000E-05, 8.472000E-02, 1.543000E-01, 6.012000E-00, 8.902000E-01, 9.210000E-01, 1.734000E+04, 6.921000E-04, false, 4.521923E-05, 4.592698E-03, 6.091000E-00, 6.743000E-01, 5.602000E-00}, + {1.152000E+01, 2.120000E+03, 4.568000E+04, 6.342000E-03, 9.210000E-03, 1.069000E+00, latvap, latice, 3.952000E-06, 6.596000E-05, 1.098000E-01, 3.456000E-01, 7.241000E-00, 9.102000E-01, 1.002000E-00, 1.734000E+04, 7.901000E-04, false, 5.236542E-05, 5.678873E-03, 7.231000E-00, 8.321000E-01, 6.092000E-00}, + {1.252000E+01, 3.145000E+03, 8.213000E+04, 9.290000E-03, 1.034000E-02, 1.069000E+00, latvap, latice, 1.952000E-06, 6.596000E-05, 2.340006E-01, 5.632000E-01, 8.452000E-00, 1.003000E-01, 2.145000E-00, 1.734000E+04, 9.212000E-04, false, 6.732276E-05, 7.321873E-03, 8.234000E-00, 9.023000E-01, 7.201000E-00}, + {1.352000E+01, 4.742000E+03, 1.014000E+05, 1.234000E-02, 1.456000E-02, 1.069000E+00, latvap, latice, 9.952000E-07, 6.596000E-05, 4.123000E-01, 6.128000E-01, 9.076000E-00, 2.831000E-01, 3.902000E-00, 1.734000E+04, 1.023000E-03, false, 7.902887E-05, 9.032908E-03, 9.021000E-00, 1.092000E-01, 8.096000E-00} }; // Sync to device @@ -64,7 +67,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { const Int offset = i * Spack::n; // Init pack inputs - Spack rho,temp, pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,kap,mu,sc, + Spack rho,temp, pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,kap,mu,sc, qv,qc_incld,qi_incld,ni_incld,qr_incld; Smask log_wetgrowth; @@ -78,8 +81,6 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { rhofaci[s] = self_device(vs).rhofaci; table_val_qi2qr_melting[s] = self_device(vs).table_val_qi2qr_melting; table_val_qi2qr_vent_melt[s] = self_device(vs).table_val_qi2qr_vent_melt; - latent_heat_vapor[s] = self_device(vs).latent_heat_vapor; - latent_heat_fusion[s] = self_device(vs).latent_heat_fusion; dv[s] = self_device(vs).dv; kap[s] = self_device(vs).kap; mu[s] = self_device(vs).mu; @@ -97,7 +98,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { log_wetgrowth.set(s, self_device(vs).log_wetgrowth); } - Functions::ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, + Functions::ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, dv, kap, mu, sc, qv, qc_incld, qi_incld, ni_incld, qr_incld, log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend); diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 3e340f68710a..28c6582f8a14 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -23,29 +23,31 @@ struct UnitWrap::UnitTest::TestP3IceMelting { static void ice_melting_bfb(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; // make array of input data (why not pass actual variables?). Copied 1st 4 rows 4x to fill pack size. IceMeltingData IceMelt[max_pack_size] = { //rho, T_atm, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, sc, mu, kap, qv, qi_incld,ni_incld - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05} + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05} }; // Sync to device @@ -65,7 +67,7 @@ static void ice_melting_bfb(){ const Int offset = i * Spack::n; // Init pack inputs - Spack rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend; + Spack rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { rho[s] = IceMelt_device(vs).rho; T_atm[s] = IceMelt_device(vs).T_atm; @@ -73,8 +75,6 @@ static void ice_melting_bfb(){ rhofaci[s] = IceMelt_device(vs).rhofaci; table_val_qi2qr_melting[s] = IceMelt_device(vs).table_val_qi2qr_melting; table_val_qi2qr_vent_melt[s] = IceMelt_device(vs).table_val_qi2qr_vent_melt; - latent_heat_vapor[s] = IceMelt_device(vs).latent_heat_vapor; - latent_heat_fusion[s] = IceMelt_device(vs).latent_heat_fusion; dv[s] = IceMelt_device(vs).dv; sc[s] = IceMelt_device(vs).sc; mu[s] = IceMelt_device(vs).mu; @@ -86,7 +86,7 @@ static void ice_melting_bfb(){ ni2nr_melt_tend[s] = IceMelt_device(vs).ni2nr_melt_tend; } - Functions::ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend); + Functions::ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { IceMelt_device(vs).qi2qr_melt_tend = qi2qr_melt_tend[s]; diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 02167261bf86..93c06a9054d7 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -187,6 +187,8 @@ static void run_bfb_ice_sed() static void run_bfb_homogeneous_freezing() { + constexpr Scalar latice = C::LatIce; + auto engine = setup_random_test(); HomogeneousFreezingData hfds_fortran[] = { @@ -203,6 +205,12 @@ static void run_bfb_homogeneous_freezing() for (auto& d : hfds_fortran) { const auto qsmall_r = std::make_pair(C::QSMALL/2, C::QSMALL*2); d.randomize(engine, { {d.T_atm, {C::T_homogfrz - 10, C::T_homogfrz + 10}}, {d.qc, qsmall_r}, {d.qr, qsmall_r} }); + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + for (int k=0; k::TestIceSupersatConservation { static void run_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + auto engine = setup_random_test(); IceSupersatConservationData f90_data[max_pack_size]; @@ -27,6 +30,10 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { for (auto& d : f90_data) { d.randomize(engine); d.dt = f90_data[0].dt; // hold this fixed, it is not packed data + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + d.latent_heat_sublim = latvap+latice; } // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that @@ -46,10 +53,9 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { const Int offset = i * Spack::n; // Init pack inputs - Spack cld_frac_i, latent_heat_sublim, qidep, qinuc, qv, qv_sat_i, t_atm, qi2qv_sublim_tend, qr2qv_evap_tend; + Spack cld_frac_i, qidep, qinuc, qv, qv_sat_i, t_atm, qi2qv_sublim_tend, qr2qv_evap_tend; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { cld_frac_i[s] = cxx_device(vs).cld_frac_i; - latent_heat_sublim[s] = cxx_device(vs).latent_heat_sublim; qidep[s] = cxx_device(vs).qidep; qinuc[s] = cxx_device(vs).qinuc; qv[s] = cxx_device(vs).qv; @@ -59,7 +65,7 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { qr2qv_evap_tend[s] = cxx_device(vs).qr2qv_evap_tend; } - Functions::ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, cxx_device(offset).dt, qi2qv_sublim_tend, qr2qv_evap_tend); + Functions::ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, t_atm, cxx_device(offset).dt, qi2qv_sublim_tend, qr2qv_evap_tend); // Copy spacks back into cxx_device view for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index bad0ca7014da..a804bd8756d5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -57,6 +57,8 @@ static void run_bfb_p3_main_part1() constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar sup_upper = -0.05; constexpr Scalar sup_lower = -0.1; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; P3MainPart1Data isds_fortran[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt @@ -74,6 +76,14 @@ static void run_bfb_p3_main_part1() {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, {d.qv_supersat_i, {sup_lower -.05, sup_upper + .05}}, {d.qc, qsmall_r}, {d.qr, qsmall_r}, {d.qi, qsmall_r} }); + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + for (int k=0; k(); } diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index 55501a30aa33..a1b270562819 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -25,14 +25,14 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { using physics = scream::physics::Functions; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //Start with reasonable values //============================ Spack pres(100000); Spack t_atm(270); Spack qv(1.7e-3); - Spack latent_heat_vapor(2.5e6); - Spack latent_heat_sublim(2.838e6); Scalar dt=60; Spack qidep(0); Spack qinuc(0); //dep and nuc only used together, so only fiddle with one @@ -50,8 +50,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Spack qv_sinks=qidep + qinuc; Spack qv_sources=qi2qv_sublim_tend_tmp + qr2qv_evap_tend_tmp; Spack qv_endstep=qv - qv_sinks*dt + qv_sources*dt; - Spack T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend_tmp*latent_heat_vapor*inv_cp )*dt; + Spack T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*(latvap+latice)*inv_cp + - qr2qv_evap_tend_tmp*latvap*inv_cp )*dt; Spack qsl = physics::qv_sat_dry(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice //just require index 0 since all entries are identical @@ -59,7 +59,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { REQUIRE(qv_endstep[0]>qsl[0]); // inputs to this test are testing what we want. //Now update sublim and evap tends using prevent_liq_supersaturation - Functions::prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, + Functions::prevent_liq_supersaturation(pres, t_atm, qv, dt, qidep, qinuc, qi2qv_sublim_tend_tmp, qr2qv_evap_tend_tmp); //Finally, recompute liquid saturation @@ -67,8 +67,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { qv_sinks=qidep + qinuc; qv_sources=qi2qv_sublim_tend_tmp + qr2qv_evap_tend_tmp; qv_endstep=qv - qv_sinks*dt + qv_sources*dt; - T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend_tmp*latent_heat_vapor*inv_cp )*dt; + T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*(latvap+latice)*inv_cp + - qr2qv_evap_tend_tmp*latvap*inv_cp )*dt; qsl = physics::qv_sat_dry(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice //just require index 0 since all entries are identical @@ -80,7 +80,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Spack qi2qv_sublim_tend_tmp2(1e-4); Spack qr2qv_evap_tend_tmp2(1e-4); - Functions::prevent_liq_supersaturation(pres, t_atm, qv_tmp, latent_heat_vapor, latent_heat_sublim, + Functions::prevent_liq_supersaturation(pres, t_atm, qv_tmp, dt, qidep, qinuc, qi2qv_sublim_tend_tmp2, qr2qv_evap_tend_tmp2); //just require index 0 since all entries are identical. REQUIRE( qi2qv_sublim_tend_tmp2[0] ==0 ); @@ -90,6 +90,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { static void run_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; auto engine = setup_random_test(); @@ -100,6 +102,11 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { for (auto& d : f90_data) { d.randomize(engine); d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + d.latent_heat_vapor = latvap; + d.latent_heat_sublim = latvap+latice; } // Create copies of data for use by cxx and sync it to device. Needs to happen before @@ -127,11 +134,9 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { // Init pack inputs Scalar dt; - Spack latent_heat_sublim, latent_heat_vapor, pres, qi2qv_sublim_tend, qidep, qinuc, qr2qv_evap_tend, qv, t_atm; + Spack pres, qi2qv_sublim_tend, qidep, qinuc, qr2qv_evap_tend, qv, t_atm; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { dt = cxx_device(vs).dt; //dt is scalar but PreventLiqSupersaturationData has diff val for each row. - latent_heat_sublim[s] = cxx_device(vs).latent_heat_sublim; - latent_heat_vapor[s] = cxx_device(vs).latent_heat_vapor; pres[s] = cxx_device(vs).pres; qi2qv_sublim_tend[s] = cxx_device(vs).qi2qv_sublim_tend; qidep[s] = cxx_device(vs).qidep; @@ -141,7 +146,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { t_atm[s] = cxx_device(vs).t_atm; } - Functions::prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend); + Functions::prevent_liq_supersaturation(pres, t_atm, qv, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend); // Copy spacks back into cxx_device view for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index df5b80cdc19e..3b994cfdad5d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -480,103 +480,105 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar dt = 1.8000E+03; constexpr bool do_predict_nc = true; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following P3UpdatePrognosticIceData pupidc[max_pack_size] = { {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, }; @@ -599,9 +601,9 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce const Int offset = i * Spack::n; // Init pack inputs - Spack qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, - ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, latent_heat_fusion, latent_heat_sublim, + Spack qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, + qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, + ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, rho_qm_cloud, th_atm, qv, qc, nc, qr, nr, qi, ni, qm, bm; Scalar dt; bool do_predict_nc; @@ -633,8 +635,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce ni_sublim_tend[s] = pupidc_device(vs).ni_sublim_tend; qc2qi_berg_tend[s] = pupidc_device(vs).qc2qi_berg_tend; inv_exner[s] = pupidc_device(vs).inv_exner; - latent_heat_fusion[s] = pupidc_device(vs).latent_heat_fusion; - latent_heat_sublim[s] = pupidc_device(vs).latent_heat_sublim; rho_qm_cloud[s] = pupidc_device(vs).rho_qm_cloud; th_atm[s] = pupidc_device(vs).th_atm; @@ -654,7 +654,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce Functions::update_prognostic_ice(qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend,ncshdc, qr2qi_collect_tend, nr_collect_tend, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, ni_nucleat_tend, - ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, latent_heat_sublim, latent_heat_fusion, + ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, do_predict_nc, log_wetgrowth, dt, pupidc_device(0).nmltratio, rho_qm_cloud, th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr); @@ -705,26 +705,28 @@ template struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables { static void get_time_space_phys_variables_unit_bfb_tests(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following GetTimeSpacePhysVarsData gtspvd[max_pack_size] = { // T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i - {2.9792E+02, 9.8711E+04, 1.1532E+00, 2.5010E+06, 2.8347E+06, 2.0321E-02, 2.0321E-02}, - {2.9792E+02, 9.8711E+04, 1.1532E+00, 2.5010E+06, 2.8347E+06, 2.0321E-02, 2.0321E-02}, - {2.9583E+02, 9.7322E+04, 1.1449E+00, 2.5010E+06, 2.8347E+06, 1.8120E-02, 1.8120E-02}, - {2.9375E+02, 9.5933E+04, 1.1366E+00, 2.5010E+06, 2.8347E+06, 1.6134E-02, 1.6134E-02}, - {2.8959E+02, 9.3156E+04, 1.1196E+00, 2.5010E+06, 2.8347E+06, 1.2729E-02, 1.2729E-02}, - {2.8750E+02, 9.1767E+04, 1.1109E+00, 2.5010E+06, 2.8347E+06, 1.1279E-02, 1.1279E-02}, - {2.8542E+02, 9.0378E+04, 1.1020E+00, 2.5010E+06, 2.8347E+06, 9.9759E-03, 9.9759E-03}, - {2.8334E+02, 8.8989E+04, 1.0931E+00, 2.5010E+06, 2.8347E+06, 8.8076E-03, 8.8076E-03}, - {2.8125E+02, 8.7600E+04, 1.0840E+00, 2.5010E+06, 2.8347E+06, 7.7615E-03, 7.7615E-03}, - {2.7917E+02, 8.6211E+04, 1.0748E+00, 2.5010E+06, 2.8347E+06, 6.8265E-03, 6.8265E-03}, - {2.7709E+02, 8.4822E+04, 1.0654E+00, 2.5010E+06, 2.8347E+06, 5.9921E-03, 5.9921E-03}, - {2.7501E+02, 8.3433E+04, 1.0559E+00, 2.5010E+06, 2.8347E+06, 5.2488E-03, 5.2488E-03}, - {2.7292E+02, 8.2044E+04, 1.0463E+00, 2.5010E+06, 2.8347E+06, 4.5879E-03, 4.5766E-03}, - {2.7084E+02, 8.0656E+04, 1.0365E+00, 2.5010E+06, 2.8347E+06, 4.0015E-03, 3.9112E-03}, - {2.6876E+02, 7.9267E+04, 1.0265E+00, 2.5010E+06, 2.8347E+06, 3.4821E-03, 3.3349E-03}, - {2.6667E+02, 7.7878E+04, 1.0164E+00, 2.5010E+06, 2.8347E+06, 3.0231E-03, 2.8368E-03}, + {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, + {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, + {2.9583E+02, 9.7322E+04, 1.1449E+00, latvap, latvap+latice, 1.8120E-02, 1.8120E-02}, + {2.9375E+02, 9.5933E+04, 1.1366E+00, latvap, latvap+latice, 1.6134E-02, 1.6134E-02}, + {2.8959E+02, 9.3156E+04, 1.1196E+00, latvap, latvap+latice, 1.2729E-02, 1.2729E-02}, + {2.8750E+02, 9.1767E+04, 1.1109E+00, latvap, latvap+latice, 1.1279E-02, 1.1279E-02}, + {2.8542E+02, 9.0378E+04, 1.1020E+00, latvap, latvap+latice, 9.9759E-03, 9.9759E-03}, + {2.8334E+02, 8.8989E+04, 1.0931E+00, latvap, latvap+latice, 8.8076E-03, 8.8076E-03}, + {2.8125E+02, 8.7600E+04, 1.0840E+00, latvap, latvap+latice, 7.7615E-03, 7.7615E-03}, + {2.7917E+02, 8.6211E+04, 1.0748E+00, latvap, latvap+latice, 6.8265E-03, 6.8265E-03}, + {2.7709E+02, 8.4822E+04, 1.0654E+00, latvap, latvap+latice, 5.9921E-03, 5.9921E-03}, + {2.7501E+02, 8.3433E+04, 1.0559E+00, latvap, latvap+latice, 5.2488E-03, 5.2488E-03}, + {2.7292E+02, 8.2044E+04, 1.0463E+00, latvap, latvap+latice, 4.5879E-03, 4.5766E-03}, + {2.7084E+02, 8.0656E+04, 1.0365E+00, latvap, latvap+latice, 4.0015E-03, 3.9112E-03}, + {2.6876E+02, 7.9267E+04, 1.0265E+00, latvap, latvap+latice, 3.4821E-03, 3.3349E-03}, + {2.6667E+02, 7.7878E+04, 1.0164E+00, latvap, latvap+latice, 3.0231E-03, 2.8368E-03}, }; // Sync to device @@ -745,14 +747,12 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables const Int offset = i * Spack::n; // Init pack inputs - Spack T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; + Spack T_atm, pres, rho, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { T_atm[s] = gtspvd_device(vs).T_atm; pres[s] = gtspvd_device(vs).pres; rho[s] = gtspvd_device(vs).rho; - latent_heat_vapor[s] = gtspvd_device(vs).latent_heat_vapor; - latent_heat_sublim[s] = gtspvd_device(vs).latent_heat_sublim; qv_sat_l[s] = gtspvd_device(vs).qv_sat_l; qv_sat_i[s] = gtspvd_device(vs).qv_sat_i; @@ -767,7 +767,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables eii[s] = gtspvd_device(vs).eii; } - Functions::get_time_space_phys_variables(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, + Functions::get_time_space_phys_variables(T_atm, pres, rho, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii); // Copy results back into views @@ -775,8 +775,6 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables gtspvd_device(vs).T_atm = T_atm[s]; gtspvd_device(vs).pres = pres[s]; gtspvd_device(vs).rho = rho[s]; - gtspvd_device(vs).latent_heat_vapor = latent_heat_vapor[s]; - gtspvd_device(vs).latent_heat_sublim = latent_heat_sublim[s]; gtspvd_device(vs).qv_sat_l = qv_sat_l[s]; gtspvd_device(vs).qv_sat_i = qv_sat_i[s]; @@ -820,72 +818,73 @@ template struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq { static void update_prognostic_liquid_unit_bfb_tests(){ + constexpr Scalar latvap = C::LatVap; //fortran generated data is input to the following P3UpdatePrognosticLiqData pupldc[max_pack_size] = { {1.0631E-12, 1.0631E+00, 1.5833E-12, 1.5833E+00, 2.4190E-02, 0.0000E+00, 0.0000E+00, 0.0000E+00, 4.2517E+00, - true , true , 8.6718E-01, 1.0037E+00, 2.5010E+06, 1.8000E+03, 2.9902E+02, 5.0000E-02, 1.0000E-06, 1.0000E+06, 1.0010E-06, + true , true , 8.6718E-01, 1.0037E+00, latvap, 1.8000E+03, 2.9902E+02, 5.0000E-02, 1.0000E-06, 1.0000E+06, 1.0010E-06, 6.3726E+05}, {3.2784E-08, 1.8780E+07, 2.1753E-11, 1.2461E+04, 7.8657E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8748E+04, - true , true , 9.8387E-01, 1.0741E+00, 2.5010E+06, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, + true , true , 9.8387E-01, 1.0741E+00, latvap, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, -8.6159E+07}, {3.2796E-09, 1.8778E+07, 1.8830E-12, 1.0782E+04, 6.8061E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.3698E+04, - true , true , 9.0740E-01, 1.0293E+00, 2.5010E+06, 1.8000E+03, 2.9376E+02, 5.0000E-03, 5.9067E-06,-6.9543E+09, 1.0439E-04, + true , true , 9.0740E-01, 1.0293E+00, latvap, 1.8000E+03, 2.9376E+02, 5.0000E-03, 5.9067E-06,-6.9543E+09, 1.0439E-04, -1.6967E+07}, {6.5634E-09, 1.8778E+07, 3.8238E-12, 1.0940E+04, 6.9061E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.3181E+04, - true , true , 9.1484E-01, 1.0339E+00, 2.5010E+06, 1.8000E+03, 2.9291E+02, 5.0000E-03, 1.1821E-05,-6.9282E+09, 1.0615E-04, + true , true , 9.1484E-01, 1.0339E+00, latvap, 1.8000E+03, 2.9291E+02, 5.0000E-03, 1.1821E-05,-6.9282E+09, 1.0615E-04, -2.8223E+07}, {9.8516E-09, 1.8779E+07, 5.8258E-12, 1.1105E+04, 7.0101E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.2655E+04, - true , true , 9.2251E-01, 1.0386E+00, 2.5010E+06, 1.8000E+03, 2.9206E+02, 5.0000E-03, 1.7743E-05,-6.9009E+09, 1.0790E-04, + true , true , 9.2251E-01, 1.0386E+00, latvap, 1.8000E+03, 2.9206E+02, 5.0000E-03, 1.7743E-05,-6.9009E+09, 1.0790E-04, -3.9628E+07}, {1.3145E-08, 1.8779E+07, 7.8929E-12, 1.1276E+04, 7.1180E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.2122E+04, - true , true , 9.3043E-01, 1.0433E+00, 2.5010E+06, 1.8000E+03, 2.9123E+02, 5.0000E-03, 2.3674E-05,-6.8725E+09, 1.0963E-04, + true , true , 9.3043E-01, 1.0433E+00, latvap, 1.8000E+03, 2.9123E+02, 5.0000E-03, 2.3674E-05,-6.8725E+09, 1.0963E-04, -5.1189E+07}, {1.6443E-08, 1.8779E+07, 1.0029E-11, 1.1454E+04, 7.2303E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.1581E+04, - true , true , 9.3860E-01, 1.0482E+00, 2.5010E+06, 1.8000E+03, 2.9040E+02, 5.0000E-03, 2.9615E-05,-6.8428E+09, 1.1136E-04, + true , true , 9.3860E-01, 1.0482E+00, latvap, 1.8000E+03, 2.9040E+02, 5.0000E-03, 2.9615E-05,-6.8428E+09, 1.1136E-04, -6.2915E+07}, {1.9746E-08, 1.8779E+07, 1.2238E-11, 1.1639E+04, 7.3471E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.1031E+04, - true , true , 9.4705E-01, 1.0531E+00, 2.5010E+06, 1.8000E+03, 2.8958E+02, 5.0000E-03, 3.5565E-05,-6.8117E+09, 1.1308E-04, + true , true , 9.4705E-01, 1.0531E+00, latvap, 1.8000E+03, 2.8958E+02, 5.0000E-03, 3.5565E-05,-6.8117E+09, 1.1308E-04, -7.4813E+07}, {2.3047E-08, 1.8779E+07, 1.4521E-11, 1.1832E+04, 7.4688E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.0474E+04, - true , true , 9.5579E-01, 1.0582E+00, 2.5010E+06, 1.8000E+03, 2.8941E+02, 4.7949E-03, 4.1510E-05,-6.7792E+09, 1.4787E-05, + true , true , 9.5579E-01, 1.0582E+00, latvap, 1.8000E+03, 2.8941E+02, 4.7949E-03, 4.1510E-05,-6.7792E+09, 1.4787E-05, -8.2885E+07}, {2.6289E-08, 1.8779E+07, 1.6845E-11, 1.2033E+04, 7.5955E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.9907E+04, - true , true , 9.6483E-01, 1.0634E+00, 2.5010E+06, 1.8000E+03, 2.8972E+02, 4.4341E-03, 4.7350E-05,-6.7452E+09,-4.7350E-05, + true , true , 9.6483E-01, 1.0634E+00, latvap, 1.8000E+03, 2.8972E+02, 4.4341E-03, 4.7350E-05,-6.7452E+09,-4.7350E-05, -8.3634E+07}, {2.9533E-08, 1.8779E+07, 1.9253E-11, 1.2242E+04, 7.7277E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.9332E+04, - true , false , 9.7418E-01, 1.0686E+00, 2.5010E+06, 1.8000E+03, 2.9002E+02, 4.0751E-03, 5.3194E-05,-6.7096E+09,-5.3194E-05, + true , false , 9.7418E-01, 1.0686E+00, latvap, 1.8000E+03, 2.9002E+02, 4.0751E-03, 5.3194E-05,-6.7096E+09,-5.3194E-05, -8.4862E+07}, {3.2784E-08, 1.8780E+07, 2.1753E-11, 1.2461E+04, 7.8657E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8748E+04, - true , false , 9.8387E-01, 1.0741E+00, 2.5010E+06, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, + true , false , 9.8387E-01, 1.0741E+00, latvap, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, -8.6159E+07}, {3.6045E-08, 1.8780E+07, 2.4356E-11, 1.2689E+04, 8.0098E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8154E+04, - true , false , 9.9391E-01, 1.0796E+00, 2.5010E+06, 1.8000E+03, 2.9063E+02, 3.3756E-03, 6.4925E-05,-6.6333E+09,-6.4925E-05, + true , false , 9.9391E-01, 1.0796E+00, latvap, 1.8000E+03, 2.9063E+02, 3.3756E-03, 6.4925E-05,-6.6333E+09,-6.4925E-05, -8.7530E+07}, {3.9321E-08, 1.8780E+07, 2.7069E-11, 1.2928E+04, 8.1605E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.7552E+04, - true , false , 1.0043E+00, 1.0853E+00, 2.5010E+06, 1.8000E+03, 2.9092E+02, 3.0417E-03, 7.0827E-05,-6.5924E+09,-7.0827E-05, + true , false , 1.0043E+00, 1.0853E+00, latvap, 1.8000E+03, 2.9092E+02, 3.0417E-03, 7.0827E-05,-6.5924E+09,-7.0827E-05, -8.8982E+07}, {4.2614E-08, 1.8780E+07, 2.9903E-11, 1.3178E+04, 8.3182E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.6939E+04, - true , false , 1.0151E+00, 1.0911E+00, 2.5010E+06, 1.8000E+03, 2.9119E+02, 2.7224E-03, 7.6760E-05,-6.5494E+09,-7.6760E-05, + true , false , 1.0151E+00, 1.0911E+00, latvap, 1.8000E+03, 2.9119E+02, 2.7224E-03, 7.6760E-05,-6.5494E+09,-7.6760E-05, -9.0523E+07}, {4.5927E-08, 1.8780E+07, 3.2867E-11, 1.3440E+04, 8.4833E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.6317E+04, - true , false , 1.0263E+00, 1.0970E+00, 2.5010E+06, 1.8000E+03, 2.9143E+02, 2.4202E-03, 8.2728E-05,-6.5044E+09,-8.2728E-05, + true , false , 1.0263E+00, 1.0970E+00, latvap, 1.8000E+03, 2.9143E+02, 2.4202E-03, 8.2728E-05,-6.5044E+09,-8.2728E-05, -9.0778E+07}, }; @@ -908,7 +907,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq // Init pack inputs Spack qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, inv_rho, - inv_exner, latent_heat_vapor, th_atm, qv, qc, nc, qr, nr; + inv_exner, th_atm, qv, qc, nc, qr, nr; bool do_predict_nc, do_prescribed_CCN; Scalar dt; @@ -929,7 +928,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq nr_selfcollect_tend[s] = pupldc_device(vs).nr_selfcollect_tend; inv_rho[s] = pupldc_device(vs).inv_rho; inv_exner[s] = pupldc_device(vs).inv_exner; - latent_heat_vapor[s] = pupldc_device(vs).latent_heat_vapor; th_atm[s] = pupldc_device(vs).th_atm; qv[s] = pupldc_device(vs).qv; @@ -941,7 +939,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq Functions::update_prognostic_liquid(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, - latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr); + dt, th_atm, qv, qc, nc, qr, nr); // Copy results back into views pupldc_device(0).dt = dt; @@ -959,7 +957,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq pupldc_device(vs).nr_selfcollect_tend = nr_selfcollect_tend[s]; pupldc_device(vs).inv_rho = inv_rho[s]; pupldc_device(vs).inv_exner = inv_exner[s]; - pupldc_device(vs).latent_heat_vapor = latent_heat_vapor[s]; pupldc_device(vs).th_atm = th_atm[s]; pupldc_device(vs).qv = qv[s]; From 3a85062a5d11f0353705be47ed8902388dd5d390 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 3 Sep 2024 13:14:25 -0600 Subject: [PATCH 400/477] Use kokkos scratch pad for local per-team bools --- .../eamxx/src/physics/p3/impl/p3_main_impl.hpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index a36a79364044..964ec21be700 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -87,9 +87,11 @@ ::p3_main_internal( const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; + using ScratchViewType = Kokkos::View; const Int nk_pack = ekat::npack(nk); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + const auto scratch_size = ScratchViewType::shmem_size(2); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack).set_scratch_size(0, Kokkos::PerTeam(scratch_size)); // load constants into local vars const Scalar inv_dt = 1 / infrastructure.dt; @@ -98,9 +100,6 @@ ::p3_main_internal( const Int kbot = kdir == -1 ? nk-1 : 0; constexpr bool debug_ABORT = false; - // per-column bools - view_2d bools("bools", nj, 2); - // we do not want to measure init stuff auto start = std::chrono::steady_clock::now(); @@ -201,9 +200,11 @@ ::p3_main_internal( const auto oqv_prev = ekat::subview(diagnostic_inputs.qv_prev, i); const auto ot_prev = ekat::subview(diagnostic_inputs.t_prev, i); - // Need to watch out for race conditions with these shared variables - bool &nucleationPossible = bools(i, 0); - bool &hydrometeorsPresent = bools(i, 1); + // Use Kokkos' scratch pad for allocating 2 bools + // per team to determine early exits + ScratchViewType bools(team.team_scratch(0), 2); + bool &nucleationPossible = bools(0); + bool &hydrometeorsPresent = bools(1); view_1d_ptr_array zero_init = { &mu_r, &lamr, &logn0r, &nu, &cdist, &cdist1, &cdistr, From cbf149b928e4e804cd478d5df4a7386362da3a45 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 3 Sep 2024 14:21:37 -0500 Subject: [PATCH 401/477] Hommexx: Use portable PRIx64 instead of lx in printf. --- .../homme/src/share/cxx/utilities/InternalDiagnostics.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp b/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp index 3249966c0b5e..ffc9657f4108 100644 --- a/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp +++ b/components/homme/src/share/cxx/utilities/InternalDiagnostics.cpp @@ -13,6 +13,8 @@ #include "TimeLevel.hpp" #include "mpi/Comm.hpp" +#include + namespace Homme { namespace { @@ -50,10 +52,10 @@ void print_global_state_hash (const std::string& label) { all_reduce_HashType(comm.mpi_comm(), accum, gaccum, 5); if (comm.root()) { for (int i = 0; i < NUM_TIME_LEVELS; ++i) - fprintf(stderr, "hxxhash> %14d %1d %16lx (E %s)\n", + fprintf(stderr, "hxxhash> %14d %1d %16" PRIx64 " (E %s)\n", tl.nstep, i, gaccum[i], label.c_str()); for (int i = 0; i < Q_NUM_TIME_LEVELS; ++i) - fprintf(stderr, "hxxhash> %14d %1d %16lx (T %s)\n", + fprintf(stderr, "hxxhash> %14d %1d %16" PRIx64 " (T %s)\n", tl.nstep, i, gaccum[NUM_TIME_LEVELS+i], label.c_str()); } } From 26a70a5b4db09539acc25bbd873862d7189a7228 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 5 Sep 2024 16:19:58 -0400 Subject: [PATCH 402/477] Fix frontier builds One function that was being called from kernels was missing KOKKOS_INLINE_FUNCTION. --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b8ac7de768db..64ae65714c59 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -90,6 +90,7 @@ srfEmissFunctions::create_srfEmiss_data_reader( template template +KOKKOS_INLINE_FUNCTION ScalarX srfEmissFunctions::linear_interp(const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { From 74a090ce88ae02645ef2b1a9c912dccc4372b590 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 17:07:50 -0600 Subject: [PATCH 403/477] EAMxx: call MPI_Abort in CIME runs when an exception is thrown inside EAMxx This ensures that the whole job is terminated. The MPI standard does not guarantee that an uncaught exception will cause the whole MPI job to terminate. --- .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index e9129e6fcea3..006bea1bef6a 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -57,10 +57,15 @@ void fpe_guard_wrapper (const Lambda& f) { try { f(); } catch (std::exception &e) { + // Print exception msg, then call MPI_Abort fprintf(stderr, "%s\n", e.what()); + + // Get raw comm before cleaning up singleton auto& c = ScreamContext::singleton(); + auto raw_comm = c.get().mpi_comm(); c.clean_up(); - throw; + + MPI_Abort (raw_comm,1); } // Restore the FPE flag as it was when control was handed to us. From 70222af46f2d41d247ce0cda4c327c439b7e6e04 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 17:30:34 -0600 Subject: [PATCH 404/477] EAMxx: in the fpe_guard_wrapper for CIME runs, capture all exceptions --- .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 006bea1bef6a..3e1514b38651 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -65,6 +65,14 @@ void fpe_guard_wrapper (const Lambda& f) { auto raw_comm = c.get().mpi_comm(); c.clean_up(); + MPI_Abort (raw_comm,1); + } catch (...) { + + // Get raw comm before cleaning up singleton + auto& c = ScreamContext::singleton(); + auto raw_comm = c.get().mpi_comm(); + c.clean_up(); + MPI_Abort (raw_comm,1); } From 2a4d9610f84e603a61eb0d1f81dfd04f5e9147e7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 17:59:10 -0600 Subject: [PATCH 405/477] Temporarily point to a COSP submodule branch with a bug fix We will revert once it gets properly integrated in COSP repo, and the COSP submodule is updated in E3SM. --- .gitmodules | 4 ++-- components/eam/src/physics/cosp2/external | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 69d56e140361..b36d90a00a42 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,8 +22,8 @@ branch = scorpio_classic [submodule "cosp2"] path = components/eam/src/physics/cosp2/external - url = git@github.com:CFMIP/COSPv2.0.git - branch = CESM_v2.1.4 + url = git@github.com:bartgol/COSPv2.0.git + branch = bartgol/fix-cosp_optical_inputs [submodule "cime"] path = cime url = git@github.com:ESMCI/cime.git diff --git a/components/eam/src/physics/cosp2/external b/components/eam/src/physics/cosp2/external index 9d910acba3e3..2deb41975faa 160000 --- a/components/eam/src/physics/cosp2/external +++ b/components/eam/src/physics/cosp2/external @@ -1 +1 @@ -Subproject commit 9d910acba3e3a3151de231184d4b109f65e28aee +Subproject commit 2deb41975faa4f5eacfc8b8f12b85acf6583d407 From c346b79fe9162b73d897e9ace9e3e6d1a8d05935 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 6 Sep 2024 14:22:08 -0600 Subject: [PATCH 406/477] Test fix --- .../single-process/rrtmgp/rrtmgp_standalone_unit.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 75feeb1b8dd4..dc1ece59b287 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -422,9 +422,14 @@ TEST_CASE("rrtmgp_scream_standalone", "") { #endif #ifdef RRTMGP_ENABLE_KOKKOS TEST_CASE("rrtmgp_scream_standalone_k", "") { - using interface_t = scream::rrtmgp::rrtmgp_interface<>; +#ifdef RRTMGP_LAYOUT_LEFT + using layout_t = Kokkos::LayoutLeft; +#else + using layout_t = typename ekat::KokkosTypes::Layout; +#endif + using interface_t = scream::rrtmgp::rrtmgp_interface; using MDRP = interface_t::MDRP; - using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using utils_t = rrtmgpTest::rrtmgp_test_utils; using real1dk = interface_t::view_t; using real2dk = interface_t::view_t; using real3dk = interface_t::view_t; From a8019dc9b3b2d6b18ab1871e991c373638dfc9f2 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 9 Sep 2024 17:59:35 -0500 Subject: [PATCH 407/477] build error when built without MOABCOMP --- components/eamxx/src/mct_coupling/atm_comp_mct.F90 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index ca7877a7bc3a..5e1b762270b2 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -308,10 +308,13 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) #ifdef MOABCOMP use mct_mod use seq_comm_mct, only : num_moab_exports +#endif + integer :: ent_type +#ifdef MOABCOMP real(r8) :: difference type(mct_list) :: temp_list - integer :: size_list, index_list, ent_type + integer :: size_list, index_list type(mct_string) :: mctOStr ! character(CXX) :: mct_field, modelStr #endif From 7bca46932236ffaa4af1287358d7e03613996501 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 9 Sep 2024 21:39:42 -0500 Subject: [PATCH 408/477] DPxx: Remove unused use_moisture line. --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 74579df552bd..cf70dd49f564 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -268,8 +268,6 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); - // Load data from IOP files, if necessary m_iop->read_iop_file_data(timestamp()); From 726807322cd42c0d12737c7b5e14a4bb4f5a1194 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 10 Sep 2024 20:34:19 -0500 Subject: [PATCH 409/477] EAMxx: Improve rrtmgp frequency err msg in buildnml.py --- components/eamxx/cime_config/eamxx_buildnml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 121506f7eff3..3e8c4abe9e95 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -181,7 +181,8 @@ def perform_consistency_checks(case, xml): pass elif rest_opt in ["nsteps", "nstep"]: expect (rest_n % rad_freq == 0, - "rrtmgp::rad_frequency incompatible with restart frequency.\n" + f"rrtmgp::rad_frequency ({rad_freq} steps) incompatible with " + f"restart frequency ({rest_n} steps).\n" " Please, ensure restart happens on a step when rad is ON") elif rest_opt in ["nseconds", "nsecond", "nminutes", "nminute", "nhours", "nhour"]: if rest_opt in ["nseconds", "nsecond"]: From 85df9de1fe0ad764722e04164005134448f98f1f Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 10 Sep 2024 21:23:41 -0500 Subject: [PATCH 410/477] EAMxx: In planar geometry case, don't scale 'lat' and 'lon'. --- .../src/dynamics/homme/interface/dyn_grid_mod.F90 | 11 +++++++++-- .../src/dynamics/homme/interface/phys_grid_mod.F90 | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 index cf8d955fe902..3460feab2fe2 100644 --- a/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 @@ -51,6 +51,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) use edge_mod_base, only: edgeVpack_nlyr, edgeVunpack_nlyr use kinds, only: real_kind, int_kind use dof_mod, only: genLocalDof + use control_mod, only: geometry ! ! Inputs ! @@ -62,6 +63,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) real(kind=real_kind), allocatable :: el_cg_gids (:,:,:) ! Homme's bex stuff only works with reals integer(kind=int_kind), allocatable :: el_dg_gids (:,:,:) ! Homme's getLocalDof might not work with c_int integer :: idof, ip,jp, ie, icol + logical :: is_sphere ! Get the gids allocate(el_cg_gids(np,np,nelemd)) @@ -78,6 +80,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) call edgeVpack_nlyr(edge,elem(ie)%desc,el_cg_gids(:,:,ie),1,0,1) enddo call bndry_exchangeV(par,edge) + is_sphere = trim(geometry) /= 'plane' do ie=1,nelemd call edgeVunpack_nlyr(edge,elem(ie)%desc,el_cg_gids(:,:,ie),1,0,1) elgids(ie) = elem(ie)%GlobalId @@ -86,8 +89,12 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) idof = (ie-1)*16+(jp-1)*4+ip cg_gids(idof) = INT(el_cg_gids(ip,jp,ie),kind=c_int) dg_gids(idof) = INT(el_dg_gids(ip,jp,ie),kind=c_int) - lat(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lat * 180.0_c_double/pi - lon(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lon * 180.0_c_double/pi + lat(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lat + lon(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lon + if (is_sphere) then + lat(ip,jp,ie) = lat(ip,jp,ie) * 180.0_c_double/pi + lon(ip,jp,ie) = lon(ip,jp,ie) * 180.0_c_double/pi + end if elgpgp(1,idof) = ie-1 elgpgp(2,idof) = jp-1 elgpgp(3,idof) = ip-1 diff --git a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 index 4d928c800b03..cc1194aed216 100644 --- a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 @@ -325,6 +325,7 @@ end function get_num_global_columns subroutine get_my_phys_data (gids, lat, lon, area, pg_type) use homme_context_mod, only: iam use shr_const_mod, only: pi=>SHR_CONST_PI + use control_mod, only: geometry ! ! Input(s) ! @@ -336,6 +337,7 @@ subroutine get_my_phys_data (gids, lat, lon, area, pg_type) ! integer :: pgN, load_bal, idof, ndofs type(pg_specs_t), pointer :: pgs + logical :: is_sphere ! Possible values for pg_type: ! 0 : physics grid on GLL nodes @@ -364,10 +366,15 @@ subroutine get_my_phys_data (gids, lat, lon, area, pg_type) ! TODO: when you enable twin columns, you'll have to manually ! do the search, since you can't just grab the offset-ed entries ndofs = get_num_local_columns (pgN) + is_sphere = trim(geometry) /= 'plane' do idof=1,ndofs gids(idof) = pgs%g_dofs(pgs%g_dofs_offsets(iam+1) + idof) - lat(idof) = pgs%g_lat (pgs%g_dofs_offsets(iam+1) + idof) * 180.0_c_double / pi - lon(idof) = pgs%g_lon (pgs%g_dofs_offsets(iam+1) + idof) * 180.0_c_double / pi + lat(idof) = pgs%g_lat (pgs%g_dofs_offsets(iam+1) + idof) + lon(idof) = pgs%g_lon (pgs%g_dofs_offsets(iam+1) + idof) + if (is_sphere) then + lat(idof) = lat(idof) * 180.0_c_double / pi + lon(idof) = lon(idof) * 180.0_c_double / pi + end if area(idof) = pgs%g_area(pgs%g_dofs_offsets(iam+1) + idof) enddo From f08004483cae52fb217a539ff1186c33add80e3f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 11 Sep 2024 10:23:10 -0600 Subject: [PATCH 411/477] EAMxx: make rrtmgp call find_package for kokkos If an existing install of Kokkos is used, the imported target has local scope, so following calls to cmake's target_link_libraries won't recognize Kokkos if the first call to find_package was in a subdir. In EAMxx, that's the case (it's found while parsing the ekat subdir). However, subsequent calls to find_package are virtually no-op, so it doesn't hurt to call it again --- components/eamxx/src/physics/rrtmgp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index 0d42d61fa662..1d3df9b4d2e4 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -127,6 +127,7 @@ yakl_process_target(rrtmgp) # NOTE: cannot use 'PUBLIC' in target_link_libraries, # since yakl_process_target already used it # with the "plain" signature +find_package(Kokkos REQUIRED) target_link_libraries(rrtmgp yakl Kokkos::kokkos) target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL From 17aa536cc4540d52bc8a23f6ca86cf435f59d2d3 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 11 Sep 2024 15:52:34 -0600 Subject: [PATCH 412/477] EAMxx: look for kokkos pkg only if it's not a target --- components/eamxx/src/physics/rrtmgp/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index 1d3df9b4d2e4..a8e47384552a 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -127,7 +127,9 @@ yakl_process_target(rrtmgp) # NOTE: cannot use 'PUBLIC' in target_link_libraries, # since yakl_process_target already used it # with the "plain" signature -find_package(Kokkos REQUIRED) +if (NOT TARGET Kokkos::kokkos) + find_package(Kokkos REQUIRED) +endif () target_link_libraries(rrtmgp yakl Kokkos::kokkos) target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL From a282adca3bb0b89d7d942307328227573b13c25e Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 13 Sep 2024 15:39:51 -0600 Subject: [PATCH 413/477] Sync lat/lon to host in rrtmgp interface for IOP lat/lon are used on both host and device in run_impl(), but when IOP is used we were changing the value without syncing to host. --- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index e89e02fe301b..9942d612f9f5 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -59,12 +59,15 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ if (m_iop) { // For IOP runs, we need to use the lat/lon from the - // IOP files instead of the geometry data. + // IOP files instead of the geometry data. Deep copy + // to device and sync to host since both will be used. m_lat = m_grid->get_geometry_data("lat").clone(); m_lat.deep_copy(m_iop->get_params().get("target_latitude")); + m_lat.sync_to_host(); m_lon = m_grid->get_geometry_data("lon").clone(); m_lon.deep_copy(m_iop->get_params().get("target_longitude")); + m_lon.sync_to_host(); } else { m_lat = m_grid->get_geometry_data("lat"); m_lon = m_grid->get_geometry_data("lon"); From 975a93feaa74891911aec8fe9ccd92b91d3fa18c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:27:44 +0000 Subject: [PATCH 414/477] Bump DavidAnson/markdownlint-cli2-action from 16 to 17 Bumps [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action) from 16 to 17. - [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases) - [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/v16...v17) --- updated-dependencies: - dependency-name: DavidAnson/markdownlint-cli2-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/e3sm-gh-md-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 507c2c3745ea..424a871637b6 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -22,7 +22,7 @@ jobs: with: files: '**/*.md' separator: "," - - uses: DavidAnson/markdownlint-cli2-action@v16 + - uses: DavidAnson/markdownlint-cli2-action@v17 if: steps.changed-files.outputs.any_changed == 'true' with: config: 'docs/.markdownlint.json' From 9eb7a7fa562efc5b75bdb987d26792fbe3f8b698 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 16 Sep 2024 13:09:56 -0700 Subject: [PATCH 415/477] bug fix when defining gsize in dp mode --- share/streams/shr_dmodel_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/streams/shr_dmodel_mod.F90 b/share/streams/shr_dmodel_mod.F90 index eea74a66cdee..e1a7c39d783e 100644 --- a/share/streams/shr_dmodel_mod.F90 +++ b/share/streams/shr_dmodel_mod.F90 @@ -383,7 +383,7 @@ subroutine shr_dmodel_readgrid( gGrid, gsMap, nxgo, nygo, nzgo, filename, compid nxgo = scm_nx nygo = scm_ny nzgo = -1 - gsize = abs(scm_nx*scm_nx*nzgo) + gsize = abs(scm_nx*scm_ny*nzgo) else nxgo = nxg nygo = nyg From c194ee2d40bd7b50f66233b8367f78fb3468f428 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 16 Sep 2024 17:09:14 -0600 Subject: [PATCH 416/477] Remove member of class from parallel_for. --- .../eamxx_mam_constituent_fluxes_interface.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index c168c1ef0e39..440e6e3dbdca 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -274,20 +274,25 @@ void MAMConstituentFluxes::run_impl(const double dt) { // Compute vertical layer heights and updraft velocity. We need these to fully // populate dry_atm_, so that we can form a HAERO atmosphere object. HAERO // atmosphere object is used to for state%q like array. + // NOTE: We cannot pass a member of the interface class (in this case, MAMConstituentFluxes) + // inside a parallel_for. Instead, we must create a soft copy of each member. + const auto & wet_atm = wet_atm_; + const auto & dry_atm= dry_atm_; + auto lambda = KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { const int icol = team.league_rank(); // column index - compute_dry_mixing_ratios(team, wet_atm_, // in - dry_atm_, // out + compute_dry_mixing_ratios(team, wet_atm, // in + dry_atm, // out icol); // in team.team_barrier(); // vertical heights has to be computed after computing dry mixing ratios // for atmosphere compute_vertical_layer_heights(team, // in - dry_atm_, // out + dry_atm, // out icol); // in - compute_updraft_velocities(team, wet_atm_, // in - dry_atm_, // out + compute_updraft_velocities(team, wet_atm, // in + dry_atm, // out icol); // in }; // policy From ba4dea01f7b51316e3ed2bee9478ed50a4f4db4a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 18 Sep 2024 09:24:59 -0600 Subject: [PATCH 417/477] EAMxx: add checks on permissions for input files at buildnml time --- .../eamxx/cime_config/eamxx_buildnml.py | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 3e8c4abe9e95..14c9cf76bea8 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -4,7 +4,7 @@ Used by buildnml. See buildnml for documetation. """ -import os, sys, re +import os, sys, re, pwd, grp, stat, getpass from collections import OrderedDict import xml.etree.ElementTree as ET @@ -953,7 +953,37 @@ def create_input_data_list_file(case,caseroot): # Only add files whose full path starts with the CIME's input data location if file_path.startswith(din_loc_root): fd.write("scream_dl_input_{} = {}\n".format(idx, file_path)) - + if os.path.exists(file_path): + if os.path.isdir(file_path): + raise IsADirectoryError(f"Input file '{file_path}' is a directory, not a regular file.") + if not os.path.isfile(file_path): + raise OSError(f"Input file '{file_path}' exists but is not a regular file.") + if not os.access(file_path,os.R_OK): + try: + file_stat = os.stat(file_path) + + # Get owner and group names + owner = pwd.getpwuid(file_stat.st_uid).pw_name + group = grp.getgrgid(file_stat.st_gid).gr_name + + # Get file permissions + permissions = stat.filemode(file_stat.st_mode) + + except Exception as e: + raise RuntimeError(f"Error retrieving file info for '{file_path}': {e}") + + curr_user = getpass.getuser() + user_info = pwd.getpwnam(curr_user) + group_ids = os.getgrouplist(curr_user, user_info.pw_gid) + curr_groups = [grp.getgrgid(gid).gr_name for gid in group_ids] + + raise PermissionError ("Input file exists but it is not readable for current user\n" + f" - file name: {file_path}\n" + f" - file owner: {owner}\n" + f" - file group: {group}\n" + f" - permissions: {permissions}\n" + f" - current user: {curr_user}\n" + f" - current user groups: {curr_groups}\n") ############################################################################### def do_cime_vars_on_yaml_output_files(case, caseroot): From b4c9aab266e2daf6481f2602c91b78bdb0d52a18 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 18 Sep 2024 14:04:43 -0600 Subject: [PATCH 418/477] EAMxx: add machine files to use with SNL-hosted containers for gh actions testing --- .../cmake/machine-files/ghci-snl-openmp.cmake | 9 +++++++++ .../eamxx/cmake/machine-files/ghci-snl.cmake | 15 +++++++++++++++ components/eamxx/scripts/machines_specs.py | 5 +++++ 3 files changed, 29 insertions(+) create mode 100644 components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake create mode 100644 components/eamxx/cmake/machine-files/ghci-snl.cmake diff --git a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake new file mode 100644 index 000000000000..0d604cbb3b28 --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake @@ -0,0 +1,9 @@ +# Common settings for our ghci images +include(${CMAKE_CURRENT_LIST_DIR}/ghci.cmake) + +# Set SCREAM_MACHINE +set(SCREAM_MACHINE ghci-openmp CACHE STRING "") + +# Set OpenMP backend +set(EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) diff --git a/components/eamxx/cmake/machine-files/ghci-snl.cmake b/components/eamxx/cmake/machine-files/ghci-snl.cmake new file mode 100644 index 000000000000..f8115aec966b --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-snl.cmake @@ -0,0 +1,15 @@ +# Set Fortran flags +set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "Fortran compiler flags" FORCE) + +# Set the path to SCREAM input data +set(SCREAM_INPUT_ROOT /projects/e3sm/inputdata CACHE PATH "Path to SCREAM input data" FORCE) + +# Set the path to BLAS/LAPACK libraries +set(BLAS_LIBRARIES "/spack-installs/netlib-lapack/3.11.0/gcc/12.3.0/base/65x6uge/lib64/libblas.so" CACHE STRING "Path to BLAS library" FORCE) +set(LAPACK_LIBRARIES "/spack-installs/netlib-lapack/3.11.0/gcc/12.3.0/base/65x6uge/lib64/liblapack.so" CACHE STRING "Path to LAPACK library" FORCE) + +# Let's catch usage of code deprecated in Kokkos 4 +option (Kokkos_ENABLE_DEPRECATED_CODE_4 "" OFF) + +# We need to manage resources to spread across available cores/gpus +option (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES "" ON) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 337f8b66d222..713f49200fbe 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -93,6 +93,11 @@ "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), + "ghci-snl-openmp" : ([], + ["mpicxx","mpifort","mpicc"], + "", + "/projects/e3sm/baselines/scream/master-baselines" + ), } if pathlib.Path("~/.cime/scream_mach_specs.py").expanduser().is_file(): # pylint: disable=no-member From 07577b6ce66b3cafdbe698bc998ab68e506b3a6a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 18 Sep 2024 17:20:39 -0600 Subject: [PATCH 419/477] EAMxx: fix ghci-snl-openmp mach file --- components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake index 0d604cbb3b28..3139580e9ee0 100644 --- a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake +++ b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake @@ -1,8 +1,8 @@ # Common settings for our ghci images -include(${CMAKE_CURRENT_LIST_DIR}/ghci.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/ghci-snl.cmake) # Set SCREAM_MACHINE -set(SCREAM_MACHINE ghci-openmp CACHE STRING "") +set(SCREAM_MACHINE ghci-snl-openmp CACHE STRING "") # Set OpenMP backend set(EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) From 2bd7413743d9d9ea27eea861b44ced532f7e8a83 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 19 Sep 2024 14:39:41 -0600 Subject: [PATCH 420/477] Update EKAT submodule Brings in new remote for yaml-cpp --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index 9ffec9a4fb4c..38a533c4a72f 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 9ffec9a4fb4c03596b7e1729241b6348aad70ec1 +Subproject commit 38a533c4a72f1fdf9736163f91dbea63fbaa4653 From 284223881d0101084cabc24a02befd9903e13780 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Sep 2024 14:47:45 -0600 Subject: [PATCH 421/477] Fixes for mappy due to rebuild --- components/eamxx/cmake/machine-files/mappy.cmake | 1 - components/eamxx/scripts/jenkins/mappy_setup | 5 +---- components/eamxx/scripts/machines_specs.py | 3 +-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/components/eamxx/cmake/machine-files/mappy.cmake b/components/eamxx/cmake/machine-files/mappy.cmake index 29fb2e74b8a1..976db8223256 100644 --- a/components/eamxx/cmake/machine-files/mappy.cmake +++ b/components/eamxx/cmake/machine-files/mappy.cmake @@ -1,5 +1,4 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -set(PYTHON_EXECUTABLE "/ascldap/users/jgfouca/packages/Python-3.8.5/bin/python3.8" CACHE STRING "" FORCE) set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) diff --git a/components/eamxx/scripts/jenkins/mappy_setup b/components/eamxx/scripts/jenkins/mappy_setup index 8796d5fa60fa..c261cb8a6829 100644 --- a/components/eamxx/scripts/jenkins/mappy_setup +++ b/components/eamxx/scripts/jenkins/mappy_setup @@ -1,8 +1,5 @@ module purge -source /projects/sems/modulefiles/utils/sems-archive-modules-init.sh -module load sems-archive-env -export PATH=/ascldap/users/jgfouca/packages/Python-3.8.5/bin:$PATH -module load sems-archive-git/2.10.1 +module load sems-git/2.42.0 source $JENKINS_SCRIPT_DIR/sandia_son_proxy diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 713f49200fbe..f1fd0c9d34ba 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -28,9 +28,8 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q rhel8 -n 4 -gpu num=4", "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 acme-gcc/11.2.0 sems-archive-git/2.10.1 acme-openmpi/4.1.4 acme-netcdf/4.7.4/acme", + "mappy" : (["module purge", "module load sems-cmake/3.27.9 sems-git/2.42.0 sems-gcc/11.4.0 sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 sems-parallel-netcdf/1.12.3 sems-openblas", "export GATOR_INITIAL_MB=4000MB", - "export PATH=/ascldap/users/jgfouca/packages/valgrind-3.22.0/bin:$PATH", ], ["mpicxx","mpifort","mpicc"], "", From 798bfa6f61672b8749c9dcccc09b7d487bb81582 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 19 Sep 2024 14:39:41 -0600 Subject: [PATCH 422/477] Update EKAT submodule Brings in new remote for yaml-cpp --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index 9ffec9a4fb4c..5e9d57c2a4df 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 9ffec9a4fb4c03596b7e1729241b6348aad70ec1 +Subproject commit 5e9d57c2a4df044a5fe4014968f02246d4dcf110 From d40dfc8cb93efc9d760c3ea021d83ae55db33662 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 19 Sep 2024 14:54:46 -0700 Subject: [PATCH 423/477] Update ekat submodule in haero submodule of scream --- externals/haero | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/haero b/externals/haero index 0dff12afaf72..e5dfce49ec73 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit 0dff12afaf72f3dff7831676260812d41dd68683 +Subproject commit e5dfce49ec734d33a42abdc0125da9c7a07ed016 From 7cdcc2cdafcdce2f23e691e75133da536fe114fb Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Sep 2024 16:00:39 -0600 Subject: [PATCH 424/477] Fix is_cuda_machine --- components/eamxx/scripts/machines_specs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index f1fd0c9d34ba..479d402e7090 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -237,9 +237,9 @@ def is_cuda_machine(machine): assert_machine_supported(machine) env_setup_raw = MACHINE_METADATA[machine][0] - env_setup_str = " ".join(env_setup_raw) + env_setup_str = " ".join(env_setup_raw).lower() - return "cuda" in env_setup_str.lower() + return ("no-cuda" not in env_setup_str and "cuda" in env_setup_str) ############################################################################### def setup_mach_env(machine, ctest_j=None): From af5bc64e3faa6f5905cfd4e7659bf1f7eccee1c6 Mon Sep 17 00:00:00 2001 From: iulian Date: Fri, 20 Sep 2024 15:49:27 -0700 Subject: [PATCH 425/477] add pm-gpu MOAB root tested on perlmutter compiled with "gnugpu" compiler environment --- cime_config/machines/config_machines.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index eeb80756bb8c..b27bc0ce19b6 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -441,6 +441,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} From 788eefa9b679060bbad516dd545da2c2a9fbbe7d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 24 Sep 2024 13:59:38 -0600 Subject: [PATCH 426/477] EAMxx: explicitly source the sems modules init script in mappy standalone testing env setup --- components/eamxx/scripts/machines_specs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 479d402e7090..af61c2bff5f9 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -28,7 +28,8 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q rhel8 -n 4 -gpu num=4", "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["module purge", "module load sems-cmake/3.27.9 sems-git/2.42.0 sems-gcc/11.4.0 sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 sems-parallel-netcdf/1.12.3 sems-openblas", + "mappy" : (["source /projects/sems/modulefiles/utils/sems-modules-init.sh", + "module purge", "module load sems-cmake/3.27.9 sems-git/2.42.0 sems-gcc/11.4.0 sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 sems-parallel-netcdf/1.12.3 sems-openblas", "export GATOR_INITIAL_MB=4000MB", ], ["mpicxx","mpifort","mpicc"], From ed30c08cb84e478033799418fc0e089461f4582d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 11:11:23 -0600 Subject: [PATCH 427/477] EAMxx: change handling of multiple matches in atmchange/atmquery Use ANY:: in the node name string to allow multiple matches --- components/eamxx/scripts/atm_manip.py | 118 ++++++++++++++------------ components/eamxx/scripts/atmchange | 13 +-- 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index fbd8381e1727..75526676975c 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -14,7 +14,6 @@ from utils import expect, run_cmd_no_fail ATMCHANGE_SEP = "-ATMCHANGE_SEP-" -ATMCHANGE_ALL = "__ALL__" ATMCHANGE_BUFF_XML_NAME = "SCREAM_ATMCHANGE_BUFFER" ############################################################################### @@ -22,40 +21,32 @@ def apply_atm_procs_list_changes_from_buffer(case, xml): ############################################################################### atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) if atmchg_buffer: - atmchgs, atmchgs_all = unbuffer_changes(case) + atmchgs = unbuffer_changes(case) - expect (len(atmchgs)==len(atmchgs_all),"Failed to unbuffer changes from SCREAM_ATMCHANGE_BUFFER") - for chg, to_all in zip(atmchgs,atmchgs_all): + for chg in atmchgs: if "atm_procs_list" in chg: - expect (not to_all, "Makes no sense to change 'atm_procs_list' for all groups") - atm_config_chg_impl(xml, chg, all_matches=False) + atm_config_chg_impl(xml, chg) ############################################################################### def apply_non_atm_procs_list_changes_from_buffer(case, xml): ############################################################################### atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) if atmchg_buffer: - atmchgs, atmchgs_all = unbuffer_changes(case) + atmchgs = unbuffer_changes(case) - expect (len(atmchgs)==len(atmchgs_all),"Failed to unbuffer changes from SCREAM_ATMCHANGE_BUFFER") - for chg, to_all in zip(atmchgs,atmchgs_all): + for chg in atmchgs: if "atm_procs_list" not in chg: - atm_config_chg_impl(xml, chg, all_matches=to_all) + atm_config_chg_impl(xml, chg) ############################################################################### -def buffer_changes(changes, all_matches=False): +def buffer_changes(changes): ############################################################################### """ Take a list of raw changes and buffer them in the XML case settings. Raw changes are what goes to atm_config_chg_impl. """ # Commas confuse xmlchange and so need to be escaped. - if all_matches: - changes_temp = [c + ATMCHANGE_ALL for c in changes] - changes_str = ATMCHANGE_SEP.join(changes_temp).replace(",",r"\,") - else: - # changes_str += f"{ATMCHANGE_SEP}--all" - changes_str = ATMCHANGE_SEP.join(changes).replace(",",r"\,") + changes_str = ATMCHANGE_SEP.join(changes).replace(",",r"\,") run_cmd_no_fail(f"./xmlchange --append {ATMCHANGE_BUFF_XML_NAME}='{changes_str}{ATMCHANGE_SEP}'") @@ -63,18 +54,15 @@ def buffer_changes(changes, all_matches=False): def unbuffer_changes(case): ############################################################################### """ - From a case, get a list of raw changes. Returns (changes, all_matches_flag) + From a case, get and return a list of raw changes """ atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) atmchgs = [] - atmchgs_all = [] for item in atmchg_buffer.split(ATMCHANGE_SEP): if item.strip(): - atmchgs_all.append(ATMCHANGE_ALL in item) - atmchgs.append(item.replace(ATMCHANGE_ALL,"").replace(r"\,", ",").strip()) - - return atmchgs, atmchgs_all + atmchgs.append(item.replace(r"\,", ",").strip()) + return atmchgs ############################################################################### def reset_buffer(): @@ -116,20 +104,64 @@ def get_xml_nodes(xml_root, name): >>> [p.tag for p in get_parents(item, parent_map)] ['root', 'sub'] """ - expect("::::" not in name, f"Invalid xml node name format, '{name}' contains ::::") + expect('::::' not in name, + "Badly formatted node name: found '::::'") + + tokens = name.split("::") + expect (tokens[-1] != '', "Input query string ends with '::'. It should end with an actual node name") + if 'ANY' in tokens: + multiple_hits_ok = True + + # Check there's only ONE 'ANY' token + expect(tokens.count('ANY') == 1, "Invalid xml node name format, multiple 'ANY' tokens found.") + + # Split tokens list into two parts: before and after 'ANY' + before_any = tokens[:tokens.index('ANY')] + after_any = tokens[tokens.index('ANY') + 1:] + expect (after_any, "Input name should not end with ANY") + + # The case where name starts with ::ANY is delicate, since before_any=[''], and this + # trips the call to get_xml_nodes. Since ANY and ::ANY are conceptually the same, + # we set before_any=[] if before_any==[''] + if before_any == ['']: + before_any = [] + + # Call get_xml_nodes with '::'.join(before_any) to get the new root + new_root = get_xml_nodes(xml_root, '::'.join(before_any)) if before_any else [xml_root] + + # Reset xml_root to new_root for the next search + xml_root = new_root[0] + + # Use new_root to find all matches for whatever comes after 'ANY::' + name = '::'.join(after_any) + else: + multiple_hits_ok = False if name.startswith("::"): prefix = "./" # search immediate children only name = name[2:] else: - prefix = ".//" # search entire tree + prefix = ".//" # search entire tree + + # Handle case without ANY try: xpath_str = prefix + name.replace("::", "/") result = xml_root.findall(xpath_str) except SyntaxError as e: expect(False, f"Invalid syntax '{name}' -> {e}") + # Note: don't check that len(result)>0, since user may be ok with 0 matches + if not multiple_hits_ok and len(result)>1: + parent_map = create_parent_map(xml_root) + error_str = f"{name} is ambiguous. Use ANY in the node path to allow multiple matches. Matches:\n" + for node in result: + parents = get_parents(node, parent_map) + name = "::".join(e.tag for e in parents) + "::" + node.tag + error_str += " " + name + "\n" + + expect(False, error_str) + return result ############################################################################### @@ -303,7 +335,7 @@ def parse_change(change): return node_name,new_value,append_this ############################################################################### -def atm_config_chg_impl(xml_root, change, all_matches=False): +def atm_config_chg_impl(xml_root, change): ############################################################################### """ >>> xml = ''' @@ -361,7 +393,7 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): >>> ################ AMBIGUOUS CHANGE ####################### >>> atm_config_chg_impl(tree,'prop1=three') Traceback (most recent call last): - SystemExit: ERROR: prop1 is ambiguous (use --all to change all matches), matches: + SystemExit: ERROR: prop1 is ambiguous. Use ANY in the node path to allow multiple matches. Matches: root::prop1 root::sub::prop1 @@ -372,7 +404,7 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): False >>> atm_config_chg_impl(tree,'sub::prop1=one') True - >>> atm_config_chg_impl(tree,'prop1=three', all_matches=True) + >>> atm_config_chg_impl(tree,'ANY::prop1=three') True >>> [item.text for item in get_xml_nodes(tree,'prop1')] ['three', 'three'] @@ -413,16 +445,6 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): expect(len(matches) > 0, f"{node_name} did not match any items") - if len(matches) > 1 and not all_matches: - parent_map = create_parent_map(xml_root) - error_str = "" - for node in matches: - parents = get_parents(node, parent_map) - name = "::".join(e.tag for e in parents) + "::" + node.tag - error_str += " " + name + "\n" - - expect(False, f"{node_name} is ambiguous (use --all to change all matches), matches:\n{error_str}") - any_change = False for node in matches: any_change |= apply_change(xml_root, node, new_value, append_this) @@ -524,25 +546,11 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= expect (print_style in ["short","full"], f"Invalid print_style '{print_style}' for print_var. Use 'full' or 'short'.") - # Get the shortest unique repr of the var name - tokens = var.split("::") - if tokens[0]=='': - tokens.pop(0) - - while len(tokens)>1: - new_name = "::".join(tokens[1:]) - matches = get_xml_nodes(xml_root, new_name) - if len(matches) > 1: - break - else: - tokens.pop(0) - - # Get node, along with all its parents (which might be used for 'full' print style) + # Get matches matches = get_xml_nodes(xml_root,var) - expect(len(matches) == 1, f"Expected one match for {var}") - node = matches[0] - print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) + for node in matches: + print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) ############################################################################### def print_all_vars(xml_root,xml_node,parent_map,curr_namespace,full,dtype,value,valid_values,print_style,indent): diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index ff886ae83507..fab03a7993bd 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -31,7 +31,7 @@ def recreate_raw_xml_file(): create_raw_xml_file(case, caseroot) ############################################################################### -def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): +def atm_config_chg(changes, reset=False, buffer_only=False): ############################################################################### if not buffer_only: expect(os.path.exists("namelist_scream.xml"), @@ -66,7 +66,7 @@ def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): root = tree.getroot() for change in changes: - this_changed = atm_config_chg_impl(root, change, all_matches) + this_changed = atm_config_chg_impl(root, change) any_change |= this_changed @@ -74,7 +74,7 @@ def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): # NOTE: if a change is wrong (e.g., typo in param name), we are still buffering it. # We have no way of checking this, unfortunately. If you get an error that is # not just syntax, your best course of action is to run atmchange --reset. - buffer_changes(changes, all_matches=all_matches) + buffer_changes(changes) if not buffer_only: recreate_raw_xml_file() @@ -103,13 +103,6 @@ OR formatter_class=argparse.ArgumentDefaultsHelpFormatter ) - parser.add_argument( - "-a", "--all", - default=False, - dest="all_matches", - action="store_true", - help="Apply change to all entries matching the name" - ) parser.add_argument( "-r", "--reset", default=False, From b49df7116486cb6b36a200f20cbe2a3d642788da Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 11:43:04 -0600 Subject: [PATCH 428/477] EAMxx: adapt testmods to new atmchange syntax --- .../scream/internal_diagnostics_level/shell_commands | 2 +- .../testdefs/testmods_dirs/scream/small_kernels/shell_commands | 2 +- .../testmods_dirs/scream/small_kernels_p3/shell_commands | 2 +- .../testmods_dirs/scream/small_kernels_shoc/shell_commands | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands index cf3ca97f6dd3..f65ccdec105e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands @@ -1,2 +1,2 @@ -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b ./xmlchange POSTRUN_SCRIPT="$CIMEROOT/../components/eamxx/scripts/check-hashes-ers" diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands index e6773dce4199..a2678e6f00c0 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands @@ -1,2 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SMALL_KERNELS On' -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands index ea0e647fdcd3..a3ec62388f09 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands @@ -1,2 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_P3_SMALL_KERNELS On' -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands index da0429652975..ca520c108584 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands @@ -1,2 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SHOC_SMALL_KERNELS On' -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b From 25fa494d327a4b260317177ba1b4c2f6ea10f454 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 11:31:13 -0600 Subject: [PATCH 429/477] EAMxx: propagate atm_manip changes to cime-nml-tests and docs --- components/eamxx/docs/user/model_input.md | 6 +++--- components/eamxx/scripts/cime-nml-tests | 26 ++++++++++------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index a0076c02f89e..44fa8688fe0c 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -148,7 +148,7 @@ scoped name, which allows `atmchange` to uniquely identify the XML node to modif $ ./atmquery homme::number_of_subcycles namelist_defaults::atmosphere_processes::homme::number_of_subcycles: 1 $ ./atmchange number_of_subcycles=10 -ERROR: number_of_subcycles is ambiguous (use --all to change all matches), matches: +ERROR: internal_diagnostics_level is ambiguous. Use ANY in the node path to allow multiple matches. Matches: namelist_defaults::atmosphere_processes::number_of_subcycles namelist_defaults::atmosphere_processes::sc_import::number_of_subcycles namelist_defaults::atmosphere_processes::homme::number_of_subcycles @@ -167,7 +167,7 @@ $ ./atmquery homme::number_of_subcycles ``` In some cases, the user may be interested in changing _all_ nodes with a given name. In that case, -the `--all` flag can be used: +you can use 'ANY' as a node name: ```shell $ ./atmquery --grep number_of_subcycles @@ -183,7 +183,7 @@ $ ./atmquery --grep number_of_subcycles p3::number_of_subcycles: 1 rrtmgp::number_of_subcycles: 1 sc_export::number_of_subcycles: 1 -$ ./atmchange --all number_of_subcycles=3 +$ ./atmchange ANY::number_of_subcycles=3 Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. $ ./atmquery --grep number_of_subcycles atmosphere_processes::number_of_subcycles: 3 diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index b128121cf95f..529e10ece709 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -63,13 +63,13 @@ class TestBuildnml(unittest.TestCase): return cases[0] if len(cases) == 1 else cases ########################################################################### - def _get_values(self, case, name, value=None, expect_equal=True, all_matches=False): + def _get_values(self, case, name, value=None, expect_equal=True): ########################################################################### """ Queries a name, optionally checking if the value matches or does not match the argument. """ - if not all_matches: + if 'ANY' not in name: orig = run_cmd_assert_result(self, f"./atmquery {name} --value", from_dir=case) if value: if expect_equal: @@ -80,7 +80,7 @@ class TestBuildnml(unittest.TestCase): return [name] else: - output = run_cmd_assert_result(self, f"./atmquery {name} --grep", from_dir=case).splitlines() + output = run_cmd_assert_result(self, f"./atmquery {name}", from_dir=case).splitlines() names = [line.rsplit(": ", maxsplit=1)[0].strip() for line in output] values = [line.rsplit(": ", maxsplit=1)[1].strip() for line in output] @@ -96,17 +96,13 @@ class TestBuildnml(unittest.TestCase): ########################################################################### def _chg_atmconfig(self, changes, case, reset=False, expect_lost=None): ########################################################################### - changes = [(item[0], item[1], False) if len(item) == 2 else item for item in changes] - - expect_lost = reset if expect_lost is None else expect_lost + expect_lost = reset if expect_lost is None else expect_lost changes_unpacked = {} - for name, value, all_matches in changes: - all_matches_opt = "-a" if all_matches else "" - - names = self._get_values(case, name, value=value, expect_equal=False, all_matches=all_matches) + for name, value in changes: + names = self._get_values(case, name, value=value, expect_equal=False) - run_cmd_assert_result(self, f"./atmchange {all_matches_opt} {name}='{value}'", from_dir=case) + run_cmd_assert_result(self, f"./atmchange {name}='{value}'", from_dir=case) for item in names: changes_unpacked[item] = value @@ -288,21 +284,21 @@ class TestBuildnml(unittest.TestCase): def test_atmchanges_on_all_matches(self): ########################################################################### """ - Test that atmchange --all works + Test that atmchange works with ANY """ case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") - self._chg_atmconfig([("enable_precondition_checks", "false", True)], case) + self._chg_atmconfig([("ANY::enable_precondition_checks", "false")], case) ########################################################################### def test_atmchanges_on_all_matches_plus_spec(self): ########################################################################### """ - Test atmchange --all followed by an atmchange of one of them + Test atmchange with ANY followed by an atmchange of one of them """ case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") - self._chg_atmconfig([("enable_precondition_checks", "false", True), + self._chg_atmconfig([("ANY::enable_precondition_checks", "false"), ("p3::enable_precondition_checks", "true")], case) ########################################################################### From e46f04df79f2e275cbf0d307b48c768bcbb4dc39 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 11:56:23 -0600 Subject: [PATCH 430/477] EAMxx: keep -a/--all in atmchange as a deprecated option * Added example in help to show how to change all matches * Deprecated --all. If used, it will print an error and exit --- components/eamxx/scripts/atmchange | 16 ++++++++++++++++ components/eamxx/scripts/atmquery | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index fab03a7993bd..c45d7cd767a4 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -23,6 +23,13 @@ sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) from standard_script_setup import * # pylint: disable=wildcard-import from CIME.case import Case +class DeprecatedAllFlag(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + error_msg = f"Error: '{option_string}' has been deprecated and should not be used.\n" + error_msg += " If you want to change ALL matches for a given var, use 'ANY::$varname=value' syntax instead." + print(error_msg, file=sys.stderr) + sys.exit(1) + ############################################################################### def recreate_raw_xml_file(): ############################################################################### @@ -96,6 +103,9 @@ OR \033[1;32m# Change param foo to 'hi' (only works if foo is unambiguous)\033[0m > {0} foo=hi + \033[1;32m# Change all matches of param foo to 'hi'\033[0m + > {0} ANY::foo=hi + \033[1;32m# Change params foo to 'hi' and append 'there' to bar (only works if both are unambiguous)\033[0m > {0} foo=hi bar+=there """.format(pathlib.Path(args[0]).name), @@ -110,6 +120,12 @@ OR help="Forget all previous atmchanges", ) + parser.add_argument( + "-a", "--all", + action=DeprecatedAction, + help="This syntax is deprecated and should not be used. Use ANY:: scope instead." + ) + parser.add_argument( "-b", "--buffer-only", default=False, diff --git a/components/eamxx/scripts/atmquery b/components/eamxx/scripts/atmquery index f07ecefa94f1..1dbfbe5f5795 100755 --- a/components/eamxx/scripts/atmquery +++ b/components/eamxx/scripts/atmquery @@ -69,7 +69,7 @@ OR "--grep", default=False, action="store_true", - help="List all matching variables and their values.", + help="List all variables matching the input regex string and their values.", ) parser.add_argument( From 9ceaa5e9c56b78933c9600a248054f1dafa0a8fb Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 12:12:17 -0600 Subject: [PATCH 431/477] EAMxx: add example for "scoped" use of ANY in user docs --- components/eamxx/docs/user/model_input.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index 44fa8688fe0c..def86e25c80f 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -157,6 +157,7 @@ ERROR: internal_diagnostics_level is ambiguous. Use ANY in the node path to allo namelist_defaults::atmosphere_processes::physics::mac_aero_mic::tms::number_of_subcycles namelist_defaults::atmosphere_processes::physics::mac_aero_mic::shoc::number_of_subcycles namelist_defaults::atmosphere_processes::physics::mac_aero_mic::cldFraction::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::mac_aero_mic::spa::internal_diagnostics_level namelist_defaults::atmosphere_processes::physics::mac_aero_mic::p3::number_of_subcycles namelist_defaults::atmosphere_processes::physics::rrtmgp::number_of_subcycles namelist_defaults::atmosphere_processes::sc_export::number_of_subcycles @@ -199,6 +200,24 @@ $ ./atmquery --grep number_of_subcycles rrtmgp::number_of_subcycles: 3 sc_export::number_of_subcycles: 3 ``` +In addition, "ANY" can be used in a "scoped" string, to limit the set of matches: +```shell +$ ./atmchange mac_aero_mic::ANY::number_of_subcycles=1 +Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. +$ ./atmquery --grep number_of_subcycles + atmosphere_processes::number_of_subcycles: 3 + sc_import::number_of_subcycles: 3 + homme::number_of_subcycles: 3 + physics::number_of_subcycles: 3 + mac_aero_mic::number_of_subcycles: 1 + tms::number_of_subcycles: 1 + shoc::number_of_subcycles: 1 + cldFraction::number_of_subcycles: 1 + spa::number_of_subcycles: 1 + p3::number_of_subcycles: 1 + rrtmgp::number_of_subcycles: 3 + sc_export::number_of_subcycles: 3 +``` Since the XML file stores constraints on the parameter value (like its type or valid values), attempting to use the wrong type will cause an error: From de37080af685d0319b808d5b81ace840d6f3f327 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 25 Sep 2024 13:19:05 -0600 Subject: [PATCH 432/477] Revert Kokkos rrtmgp There are FAILs and big performance regressions --- components/eamxx/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 8d23ee67c432..5715e5be1745 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -212,8 +212,8 @@ endif() # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) -option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE) -option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE) if (SCREAM_RRTMGP_ENABLE_YAKL) add_definitions("-DRRTMGP_ENABLE_YAKL") endif() From bc518277cbe2034aa88cccf788b670462900877f Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Wed, 25 Sep 2024 13:31:33 -0700 Subject: [PATCH 433/477] remove DPxx dt_remap_factor setting --- components/eamxx/cime_config/namelist_defaults_scream.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 3d6a86556dee..57611fc3aa91 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -659,7 +659,6 @@ be lost if SCREAM_HACK_XML is not enabled. 2 False 2 - 1 1 6 0 From 0260e85a3afb3ad44ccb91dd87e6f27557b3c28c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 25 Sep 2024 15:18:57 -0600 Subject: [PATCH 434/477] EAMxx: fix error in atmchange --- components/eamxx/scripts/atmchange | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index c45d7cd767a4..eb8de06a6931 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -122,7 +122,8 @@ OR parser.add_argument( "-a", "--all", - action=DeprecatedAction, + action=DeprecatedAllFlag, + default=argparse.SUPPRESS, help="This syntax is deprecated and should not be used. Use ANY:: scope instead." ) From 67ab8bfec0492f8f1941bde87b8fc9f8fbb76ba7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 11:50:54 -0600 Subject: [PATCH 435/477] EAMxx: do not include the xml root in the list of parents Besides not adding much, when used in cime-nml-tests, as part of finding matches to atmchange, it messes up future searches --- components/eamxx/scripts/atm_manip.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index 75526676975c..d3421ef62573 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -454,7 +454,9 @@ def atm_config_chg_impl(xml_root, change): ############################################################################### def create_parent_map(root): ############################################################################### - return {c: p for p in root.iter() for c in p} + pmap = {c: p for p in root.iter() for c in p} + pmap[root] = None + return pmap ############################################################################### def get_parents(elem, parent_map): @@ -464,9 +466,11 @@ def get_parents(elem, parent_map): be the furthest ancestor, last item will be direct parent) """ results = [] - if elem in parent_map: + if elem in parent_map and parent_map[elem] is not None: parent = parent_map[elem] - results = get_parents(parent, parent_map) + [parent] + results = get_parents(parent, parent_map) + if parent_map[parent] is not None: + results.append(parent) return results From e7a3bdb65bbbdca56a559d31d35fd01db7fd3593 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 11:51:12 -0600 Subject: [PATCH 436/477] EAMxx: allow to print non-leaf XML nodes recursively in atmquery --- components/eamxx/scripts/atm_manip.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index d3421ef62573..d555c3bf1f4d 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -478,6 +478,14 @@ def get_parents(elem, parent_map): def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="invalid",indent=""): ############################################################################### + if len(node)>0: + print (f"{indent}{node.tag}:") + # This is not a leaf, so print all nested nodes + for child in node: + print_var_impl(child,parent_map,full,dtype,value,valid_values,'short',indent+" ") + + return + expect (print_style in ["short","full"], f"Invalid print_style '{print_style}' for print_var_impl. Use 'full' or 'short'.") @@ -597,6 +605,7 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, root::prop1: one sub::prop1: two """ + parent_map = create_parent_map(xml_root) if listall: print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") @@ -619,6 +628,6 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, else: for var in variables: - print_var(xml_root,parent_map,var,full,dtype,value,valid_values,"full"," ") + print_var(xml_root,parent_map,var,full,dtype,value,valid_values,"full"," ") return True From e85bfe9237dc1a8e99fd1ed90ce6e237df4febe4 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 12:47:05 -0600 Subject: [PATCH 437/477] EAMxx: fix bug in cime-nml-tests --- components/eamxx/scripts/cime-nml-tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 529e10ece709..2ff960ca2712 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -253,7 +253,8 @@ class TestBuildnml(unittest.TestCase): Test that atmchanges via testmod are preserved """ def_mach_comp = \ - run_cmd_assert_result(self, "../CIME/Tools/list_e3sm_tests cime_tiny", from_dir=CIME_SCRIPTS_DIR).splitlines()[-1].split(".")[-1] + run_cmd_assert_result(self, "../CIME/Tools/list_e3sm_tests cime_tiny", from_dir=CIME_SCRIPTS_DIR).splitlines()[-2].split(".")[-1] + case = self._create_test(f"SMS.ne30_ne30.F2010-SCREAMv1.{def_mach_comp}.scream-scream_example_testmod_atmchange --no-build") # Check that the value match what's in the testmod From 1035a5095df2a647ea34fdfd38fc584cabde42f2 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 12:47:37 -0600 Subject: [PATCH 438/477] EAMxx: allow using ANY in atmquery as a leaf name --- components/eamxx/scripts/atm_manip.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index d555c3bf1f4d..c26eb3b3933e 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -118,7 +118,6 @@ def get_xml_nodes(xml_root, name): # Split tokens list into two parts: before and after 'ANY' before_any = tokens[:tokens.index('ANY')] after_any = tokens[tokens.index('ANY') + 1:] - expect (after_any, "Input name should not end with ANY") # The case where name starts with ::ANY is delicate, since before_any=[''], and this # trips the call to get_xml_nodes. Since ANY and ::ANY are conceptually the same, @@ -133,7 +132,7 @@ def get_xml_nodes(xml_root, name): xml_root = new_root[0] # Use new_root to find all matches for whatever comes after 'ANY::' - name = '::'.join(after_any) + name = '*' if not after_any else '::'.join(after_any) else: multiple_hits_ok = False From b5f97f8cf80e5d6be4603775657e56e0d56e658c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 12:51:03 -0600 Subject: [PATCH 439/477] EAMxx: changes for print style in atmquery Before we had two styles, 'short', and 'full'. The former only printed the leaf name, the latter printed the full scope. I changed those to 'node-name' and 'full-scope', and added a new style called 'parent-scope', which only adds one parent to the scope of the name. These types are used by default for: - node-name: when printing recursively - full-scope: when using --grep (unless the grep matches a non-leaf node, in which case the nested values are printed with node-name format) - parent-scope: the default when printing a parameter --- components/eamxx/scripts/atm_manip.py | 57 +++++++++++++-------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index c26eb3b3933e..41150d12f655 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -479,18 +479,21 @@ def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="in if len(node)>0: print (f"{indent}{node.tag}:") - # This is not a leaf, so print all nested nodes + # This is not a leaf, so print all nested nodes. for child in node: - print_var_impl(child,parent_map,full,dtype,value,valid_values,'short',indent+" ") - + # Since prints are nicely nested, use 'node-name' as print style + print_var_impl(child,{},full,dtype,value,valid_values,'node-name',indent+" ") return - expect (print_style in ["short","full"], + # print (f"printing leaf={node.tag}, len(parent_map)={len(parent_map)}") + expect (print_style in ["node-name","full-scope","parent-scope"], f"Invalid print_style '{print_style}' for print_var_impl. Use 'full' or 'short'.") - if print_style=="short": + if print_style=="node-name": # Just the inner most name name = node.tag + elif print_style=="parent-scope": + name = parent_map[node].tag + "::" + node.tag else: parents = get_parents(node, parent_map) name = "::".join(e.tag for e in parents) + "::" + node.tag @@ -538,24 +541,24 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= >>> tree = ET.fromstring(xml) >>> parent_map = create_parent_map(tree) >>> ################ Missing type data ####################### - >>> print_var(tree,parent_map,'::prop1',False,True,False,False,"short") + >>> print_var(tree,parent_map,'::prop1',False,True,False,False,"node-name") Traceback (most recent call last): SystemExit: ERROR: Error! Missing type information for prop1 - >>> print_var(tree,parent_map,'prop2',True,False,False,False,"short") + >>> print_var(tree,parent_map,'prop2',True,False,False,False,"node-name") prop2 value: 2 type: integer valid values: ['1', '2'] - >>> print_var(tree,parent_map,'prop2',False,True,False,False,"short") + >>> print_var(tree,parent_map,'prop2',False,True,False,False,"node-name") prop2: integer - >>> print_var(tree,parent_map,'prop2',False,False,True,False,"short") + >>> print_var(tree,parent_map,'prop2',False,False,True,False,"node-name") 2 - >>> print_var(tree,parent_map,'prop2',False,False,False,True,"short"," ") + >>> print_var(tree,parent_map,'prop2',False,False,False,True,"node-name"," ") prop2: ['1', '2'] """ - expect (print_style in ["short","full"], - f"Invalid print_style '{print_style}' for print_var. Use 'full' or 'short'.") + expect (print_style in ["node-name","full-scope","parent-scope"], + f"Invalid print_style '{print_style}' for print_var. Use 'node-name', 'full-scope', or 'parent-scope'.") # Get matches matches = get_xml_nodes(xml_root,var) @@ -563,20 +566,9 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= for node in matches: print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) -############################################################################### -def print_all_vars(xml_root,xml_node,parent_map,curr_namespace,full,dtype,value,valid_values,print_style,indent): -############################################################################### - - print (f"{indent}{xml_node.tag}") - for c in xml_node: - if len(c)>0: - print_all_vars(xml_root,c,parent_map,curr_namespace+c.tag+"::",full,dtype,value,valid_values,print_style,indent+" ") - else: - print_var(xml_root,parent_map,curr_namespace+c.tag,full,dtype,value,valid_values,print_style,indent+" ") - ############################################################################### def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, - dtype=False, valid_values=False, grep=False): + dtype=False, valid_values=False, grep=False,parent_map=None): ############################################################################### """ >>> xml = ''' @@ -605,28 +597,33 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, sub::prop1: two """ - parent_map = create_parent_map(xml_root) + if not parent_map: + parent_map = create_parent_map(xml_root) if listall: - print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") elif grep: for regex in variables: expect("::" not in regex, "query --grep does not support including parent info") var_re = re.compile(f'{regex}') if var_re.search(xml_root.tag): - print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") + if len(xml_root)>0: + parents = get_parents(xml_root,parent_map) + print (f"{'::'.join([p.tag for p in parents]) + '::' + xml_root.tag}:") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") else: for elem in xml_root: if len(elem)>0: - atm_query_impl(elem,variables,listall,full,value,dtype,valid_values,grep) + atm_query_impl(elem,variables,listall,full,value,dtype,valid_values,grep,parent_map) else: if var_re.search(elem.tag): nodes = get_xml_nodes(xml_root, "::"+elem.tag) expect(len(nodes) == 1, "No matches?") - print_var_impl(nodes[0],parent_map,full,dtype,value,valid_values,"full"," ") + print_var_impl(nodes[0],parent_map,full,dtype,value,valid_values,"full-scope"," ") else: for var in variables: - print_var(xml_root,parent_map,var,full,dtype,value,valid_values,"full"," ") + pmap = {} if var=='ANY' else parent_map + print_var(xml_root,pmap,var,full,dtype,value,valid_values,"parent-scope"," ") return True From fa3ff46c4a937ac59036308440dd2fa28c6185bd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 26 Sep 2024 14:35:01 -0600 Subject: [PATCH 440/477] EAMxx: fix doctests and corner-cases in atm_manip --- .../eamxx/cime_config/eamxx_buildnml.py | 4 +- components/eamxx/scripts/atm_manip.py | 99 ++++++++++++++----- components/eamxx/scripts/cime-nml-tests | 31 +++--- 3 files changed, 92 insertions(+), 42 deletions(-) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 14c9cf76bea8..bf425c2279c7 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -126,7 +126,7 @@ def perform_consistency_checks(case, xml): >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':2, 'REST_OPTION':'nsteps'}) >>> perform_consistency_checks(case,xml) Traceback (most recent call last): - CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. + CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency (3 steps) incompatible with restart frequency (2 steps). Please, ensure restart happens on a step when rad is ON >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':10800, 'REST_OPTION':'nseconds'}) >>> perform_consistency_checks(case,xml) @@ -970,7 +970,7 @@ def create_input_data_list_file(case,caseroot): permissions = stat.filemode(file_stat.st_mode) except Exception as e: - raise RuntimeError(f"Error retrieving file info for '{file_path}': {e}") + raise RuntimeError(f"Error retrieving file info for '{file_path}': {e}") from e curr_user = getpass.getuser() user_info = pwd.getpwnam(curr_user) diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index 41150d12f655..ae77db7e1826 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -89,11 +89,11 @@ def get_xml_nodes(xml_root, name): >>> ################ INVALID SYNTAX ####################### >>> get_xml_nodes(tree,'sub::::prop1') Traceback (most recent call last): - SystemExit: ERROR: Invalid xml node name format, 'sub::::prop1' contains :::: + SystemExit: ERROR: Invalid xml node name format, 'sub::::prop1' contains '::::' >>> ################ VALID USAGE ####################### >>> get_xml_nodes(tree,'invalid::prop1') [] - >>> [item.text for item in get_xml_nodes(tree,'prop1')] + >>> [item.text for item in get_xml_nodes(tree,'ANY::prop1')] ['one', 'two'] >>> [item.text for item in get_xml_nodes(tree,'::prop1')] ['one'] @@ -102,10 +102,10 @@ def get_xml_nodes(xml_root, name): >>> item = get_xml_nodes(tree,'prop2')[0] >>> parent_map = create_parent_map(tree) >>> [p.tag for p in get_parents(item, parent_map)] - ['root', 'sub'] + ['sub'] """ expect('::::' not in name, - "Badly formatted node name: found '::::'") + f"Invalid xml node name format, '{name}' contains '::::'") tokens = name.split("::") expect (tokens[-1] != '', "Input query string ends with '::'. It should end with an actual node name") @@ -156,7 +156,8 @@ def get_xml_nodes(xml_root, name): error_str = f"{name} is ambiguous. Use ANY in the node path to allow multiple matches. Matches:\n" for node in result: parents = get_parents(node, parent_map) - name = "::".join(e.tag for e in parents) + "::" + node.tag + name = "::".join(e.tag for e in parents) + name = node.tag if name=="" else name + "::" + node.tag error_str += " " + name + "\n" expect(False, error_str) @@ -393,8 +394,8 @@ def atm_config_chg_impl(xml_root, change): >>> atm_config_chg_impl(tree,'prop1=three') Traceback (most recent call last): SystemExit: ERROR: prop1 is ambiguous. Use ANY in the node path to allow multiple matches. Matches: - root::prop1 - root::sub::prop1 + prop1 + sub::prop1 >>> ################ VALID USAGE ####################### >>> atm_config_chg_impl(tree,'::prop1=two') @@ -405,7 +406,7 @@ def atm_config_chg_impl(xml_root, change): True >>> atm_config_chg_impl(tree,'ANY::prop1=three') True - >>> [item.text for item in get_xml_nodes(tree,'prop1')] + >>> [item.text for item in get_xml_nodes(tree,'ANY::prop1')] ['three', 'three'] >>> ################ TEST APPEND += ################# >>> atm_config_chg_impl(tree,'a+=4') @@ -465,14 +466,49 @@ def get_parents(elem, parent_map): be the furthest ancestor, last item will be direct parent) """ results = [] - if elem in parent_map and parent_map[elem] is not None: + if not is_root(elem,parent_map): parent = parent_map[elem] results = get_parents(parent, parent_map) - if parent_map[parent] is not None: + if not is_root(parent,parent_map): results.append(parent) return results +############################################################################### +def is_anchestor_of(parent,child,parent_map): +############################################################################### + """ + >>> xml = ''' + ... + ... one + ... + ... two + ... 2 + ... + ... + ... ''' + >>> import xml.etree.ElementTree as ET + >>> tree = ET.fromstring(xml) + >>> parent_map = create_parent_map(tree) + >>> sub = get_xml_nodes(tree,'sub')[0] + >>> sub_prop1 = get_xml_nodes(tree,'sub::prop1')[0] + >>> is_anchestor_of(sub,sub_prop1,parent_map) + True + >>> is_anchestor_of(sub_prop1,sub,parent_map) + False + """ + curr = child + while curr is not None: + if curr is parent: + return True + curr = parent_map[curr] + return False + +############################################################################### +def is_root (node,parent_map): +############################################################################### + return parent_map[node] is None + ############################################################################### def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="invalid",indent=""): ############################################################################### @@ -482,10 +518,9 @@ def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="in # This is not a leaf, so print all nested nodes. for child in node: # Since prints are nicely nested, use 'node-name' as print style - print_var_impl(child,{},full,dtype,value,valid_values,'node-name',indent+" ") + print_var_impl(child,{},full,dtype,value,valid_values,'node-name',indent+" ") return - # print (f"printing leaf={node.tag}, len(parent_map)={len(parent_map)}") expect (print_style in ["node-name","full-scope","parent-scope"], f"Invalid print_style '{print_style}' for print_var_impl. Use 'full' or 'short'.") @@ -493,10 +528,13 @@ def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="in # Just the inner most name name = node.tag elif print_style=="parent-scope": - name = parent_map[node].tag + "::" + node.tag + parent = parent_map[node] + name = node.tag if is_root(parent,parent_map) else parent.tag + "::" + node.tag else: parents = get_parents(node, parent_map) - name = "::".join(e.tag for e in parents) + "::" + node.tag + name = "::".join(e.tag for e in parents) + name += "::" if parents else "" + name += node.tag if full: expect ("type" in node.attrib.keys(), @@ -563,7 +601,19 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= # Get matches matches = get_xml_nodes(xml_root,var) - for node in matches: + # If ANY is in the var name, we may hit the case where one of the matches + # is a parent of another match. In this case, we want to get rid of + unique_matches = [] + for i in matches: + add_this = True + for j in matches: + if i is not j and is_anchestor_of(j,i,parent_map): + add_this = False + + if add_this: + unique_matches.append(i) + + for node in unique_matches: print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) ############################################################################### @@ -584,23 +634,22 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, >>> tree = ET.fromstring(xml) >>> vars = ['prop2','::prop1'] >>> success = atm_query_impl(tree, vars) - root::sub::prop2: 2 - root::prop1: one + sub::prop2: 2 + prop1: one >>> success = atm_query_impl(tree, [], listall=True, valid_values=True) - root + prop1: + sub: prop1: - sub - prop1: - prop2: ['1', '2'] + prop2: ['1', '2'] >>> success = atm_query_impl(tree,['prop1'], grep=True) - root::prop1: one + prop1: one sub::prop1: two """ if not parent_map: parent_map = create_parent_map(xml_root) if listall: - print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") elif grep: for regex in variables: @@ -610,7 +659,7 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, if len(xml_root)>0: parents = get_parents(xml_root,parent_map) print (f"{'::'.join([p.tag for p in parents]) + '::' + xml_root.tag}:") - print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") else: for elem in xml_root: if len(elem)>0: @@ -624,6 +673,6 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, else: for var in variables: pmap = {} if var=='ANY' else parent_map - print_var(xml_root,pmap,var,full,dtype,value,valid_values,"parent-scope"," ") + print_var(xml_root,pmap,var,full,dtype,value,valid_values,"parent-scope"," ") return True diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 2ff960ca2712..d5ed8e14f2e0 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -119,6 +119,7 @@ class TestBuildnml(unittest.TestCase): def setUp(self): ########################################################################### self._dirs_to_cleanup = [] + self._common_case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") ########################################################################### def tearDown(self): @@ -174,7 +175,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are not lost when eamxx setup is called """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("atm_log_level", "trace"), ("output_to_screen", "true")], case) @@ -192,7 +193,7 @@ class TestBuildnml(unittest.TestCase): """ Test that the append attribute for array-type params in namelist defaults works as expected """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Add testOnly proc self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) @@ -213,7 +214,7 @@ class TestBuildnml(unittest.TestCase): """ Test that var+=value syntax behaves as expected """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Append to an existing entry name = 'output_yaml_files' @@ -230,7 +231,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are lost when resetting """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("atm_log_level", "trace")], case, reset=True) @@ -240,7 +241,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are lost if SCREAM_HACK_XML=TRUE """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case run_cmd_assert_result(self, "./xmlchange SCREAM_HACK_XML=TRUE", from_dir=case) @@ -267,7 +268,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works when using 'namespace' syntax foo::bar """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("p3::enable_precondition_checks", "false")], case) @@ -277,7 +278,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works for array data """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("surf_mom_flux", "40.0,2.0")], case) @@ -287,7 +288,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works with ANY """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("ANY::enable_precondition_checks", "false")], case) @@ -297,7 +298,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchange with ANY followed by an atmchange of one of them """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("ANY::enable_precondition_checks", "false"), ("p3::enable_precondition_checks", "true")], case) @@ -308,7 +309,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) @@ -322,7 +323,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # modifying a procs list requires known processes or "_" pre/post suffixes stat, out, err = run_cmd ("./atmchange mac_aero_mic::atm_procs_list=shoc,cldfraction,spa,p3,spiderman", @@ -336,7 +337,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchange does not change buffer if syntax was wrong """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Attempting a bad change should not alter the content of SCREAM_ATMCHANGE_BUFFER old = run_cmd_no_fail ("./xmlquery --value SCREAM_ATMCHANGE_BUFFER",from_dir=case) @@ -353,7 +354,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchange errors out with invalid param names """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case stat, out, err = run_cmd ("./atmchange p3::non_existent=3",from_dir=case) expect (stat!=0,"Command './atmchange p3::non_existent=3' should have failed") @@ -364,7 +365,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm proc groups """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case out = run_cmd_no_fail ("./atmchange mac_aero_mic::atm_procs_list=shoc,_my_group_",from_dir=case) @@ -380,7 +381,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that remove atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa")], case) From fa54928852d3b6b8105c22c06bbe54ba97b09ba0 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 17:38:46 -0600 Subject: [PATCH 441/477] EAMxx: avoid allocating small views at runtime These were likely residues from when FieldLayout did not store device-friendly extents --- .../eamxx/src/share/field/field_impl.hpp | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 67753605864c..68f7fe6115a3 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -306,17 +306,10 @@ deep_copy_impl (const Field& src) { auto src_alloc_props = src.get_header().get_alloc_properties(); auto tgt_alloc_props = get_header().get_alloc_properties(); - // If a manual parallel_for is required (b/c of alloc sizes difference), - // we need to create extents (rather than just using the one in layout), - // since we don't know if we're running on host or device - using device_t = typename Field::get_device; - using exec_space = typename device_t::execution_space; - using RangePolicy = Kokkos::RangePolicy; - using extents_type = typename ekat::KokkosTypes::template view_1d; - extents_type ext ("",rank); - Kokkos::deep_copy(ext,layout.extents()); + using RangePolicy = typename KokkosTypes::RangePolicy; auto policy = RangePolicy(0,layout.size()); + auto ext = layout.extents(); switch (rank) { case 1: { @@ -324,14 +317,14 @@ deep_copy_impl (const Field& src) { auto v = get_view< ST*,HD>(); auto v_src = src.get_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { - v(idx) = v_src(idx); - }); + v(idx) = v_src(idx); + }); } else { auto v = get_strided_view< ST*,HD>(); auto v_src = src.get_strided_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { - v(idx) = v_src(idx); - }); + v(idx) = v_src(idx); + }); } } break; @@ -658,15 +651,10 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) " - x layout: " + x_l.to_string() + "\n" " - y layout: " + y_l.to_string() + "\n"); - using device_t = typename Field::get_device; - using exec_space = typename device_t::execution_space; - using RangePolicy = Kokkos::RangePolicy; + using RangePolicy = typename KokkosTypes::RangePolicy; + auto policy = RangePolicy(0,x_l.size()); - // Need to create extents (rather than just using the one in x_l), - // since we don't know if we're running on host or device - using extents_type = typename ekat::KokkosTypes::template view_1d; - extents_type ext ("",x_l.rank()); - Kokkos::deep_copy(ext,x_l.extents()); + auto ext = layout.extents(); auto policy = RangePolicy(0,x_l.size()); switch (x_l.rank()) { From cefaf92afc80be743bac613af5ca4ea5917ec691 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 19:49:52 -0600 Subject: [PATCH 442/477] EAMxx: always use output control specs for creating the rhist file name This avoid clashing of streams with same prefix, but different freq specs --- components/eamxx/src/share/io/scream_output_manager.cpp | 6 +++--- components/eamxx/src/share/io/scream_output_manager.hpp | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 0b738efe6483..b24b473d0b1d 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -400,7 +400,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) // Check if we need to open a new file if (not filespecs.is_open) { - filespecs.filename = compute_filename (control,filespecs,timestamp); + filespecs.filename = compute_filename (filespecs,timestamp); // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) setup_file(filespecs,control); } @@ -613,11 +613,11 @@ long long OutputManager::res_dep_memory_footprint () const { } std::string OutputManager:: -compute_filename (const IOControl& control, - const IOFileSpecs& file_specs, +compute_filename (const IOFileSpecs& file_specs, const util::TimeStamp& timestamp) const { auto filename = m_filename_prefix + file_specs.suffix(); + const auto& control = m_output_control; // Always add avg type and frequency info filename += "." + e2str(m_avg_type); diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index d25dd037c67f..4d7486380cfe 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -120,8 +120,7 @@ class OutputManager long long res_dep_memory_footprint () const; protected: - std::string compute_filename (const IOControl& control, - const IOFileSpecs& file_specs, + std::string compute_filename (const IOFileSpecs& file_specs, const util::TimeStamp& timestamp) const; void set_file_header(const IOFileSpecs& file_specs); From e540ae952b55dedb4a85bf3587b9c6a6faafeb73 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 5 Sep 2024 19:57:28 -0600 Subject: [PATCH 443/477] EAMxx: fix how we find filename in rpointer * We must consider avg type and output freq specs, to avoid name clashing * Since rhist filename contains output control specs, no need to check that freq/freq_units/avg_type are unchanged upon restart: if rhist file is found, they are ok. --- .../eamxx/src/share/io/scream_io_utils.cpp | 77 +++++++++++-------- .../eamxx/src/share/io/scream_io_utils.hpp | 11 ++- .../src/share/io/scream_output_manager.cpp | 22 ++---- .../eamxx/src/share/io/tests/io_utils.cpp | 39 +++++++--- 4 files changed, 87 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 4a0beb813b76..9ff7a641958f 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -4,6 +4,7 @@ #include "share/util/scream_utils.hpp" #include +#include namespace scream { @@ -11,37 +12,46 @@ std::string find_filename_in_rpointer ( const std::string& filename_prefix, const bool model_restart, const ekat::Comm& comm, - const util::TimeStamp& run_t0) + const util::TimeStamp& run_t0, + const OutputAvgType avg_type, + const IOControl& control) { std::string filename; bool found = false; std::string content; std::string suffix = model_restart ? ".r." : ".rhist."; + std::string pattern_str = filename_prefix + suffix; + + // The AD will pass a default constructed control, since it doesn't know the values + // of REST_N/REST_OPTION used in the previous run. Also, model restart is *always* INSTANT. + if (model_restart) { + EKAT_REQUIRE_MSG (avg_type==OutputAvgType::Instant, + "Error! Model restart output should have INSTANT avg type.\n" + " - input avg_type: " + e2str(avg_type) + "\n"); + pattern_str += e2str(OutputAvgType::Instant) + R"(.n(step|sec|min|hour|day|month|year)s_x\d+)"; + } else { + EKAT_REQUIRE_MSG (control.output_enabled(), + "Error! When restarting an output stream, we need a valid IOControl structure.\n" + " - filename prefix: " + filename_prefix + "\n"); + pattern_str += e2str(avg_type) + "." + control.frequency_units + "_x" + std::to_string(control.frequency); + } + pattern_str += "." + run_t0.to_string() + ".nc"; + std::regex pattern (pattern_str); + if (comm.am_i_root()) { std::ifstream rpointer_file; + std::string line; rpointer_file.open("rpointer.atm"); - // If the timestamp is in the filename, then the filename ends with "S.nc", - // with S being the string representation of the timestamp - auto ts_len = run_t0.to_string().size(); - auto extract_ts = [&] (const std::string& line) -> util::TimeStamp { - auto min_size = ts_len+3; - if (line.size()>=min_size) { - auto ts_str = line.substr(line.size()-min_size,ts_len); - auto ts = util::str_to_time_stamp(ts_str); - return ts; - } else { - return util::TimeStamp(); - } - }; - - while ((rpointer_file >> line) and not found) { + while (std::getline(rpointer_file,line)) { content += line + "\n"; - found = line.find(filename_prefix+suffix) != std::string::npos && - extract_ts(line)==run_t0; - filename = line; + if (std::regex_match(line,pattern)) { + filename = line; + found = true; + break; + } } } @@ -52,18 +62,23 @@ std::string find_filename_in_rpointer ( if (not found) { broadcast_string(content,comm,comm.root_rank()); - // If the history restart file is not found, it must be because the last - // model restart step coincided with a model output step, in which case - // a restart history file is not written. - // If that's the case, *disable* output restart, by setting - // 'Restart'->'Perform Restart' = false - // in the input parameter list - EKAT_ERROR_MSG ( - "Error! Restart requested, but no restart file found in 'rpointer.atm'.\n" - " restart filename prefix: " + filename_prefix + "\n" - " restart file type: " + std::string(model_restart ? "model restart" : "history restart") + "\n" - " run t0 : " + run_t0.to_string() + "\n" - " rpointer content:\n" + content); + if (model_restart) { + EKAT_ERROR_MSG ( + "Error! Restart requested, but no model restart file found in 'rpointer.atm'.\n" + " model restart filename prefix: " + filename_prefix + "\n" + " run t0 : " + run_t0.to_string() + "\n" + " rpointer content:\n" + content + "\n\n"); + } else { + EKAT_ERROR_MSG ( + "Error! Restart requested, but no history restart file found in 'rpointer.atm'.\n" + " hist restart filename prefix: " + filename_prefix + "\n" + " run t0 : " + run_t0.to_string() + "\n" + " avg_type : " + e2str(avg_type) + "\n" + " output freq : " + std::to_string(control.frequency) + "\n" + " output freq units: " + control.frequency_units + "\n" + " rpointer content:\n" + content + "\n\n" + " Did you change output specs (avg type, freq, or freq units) across restart? If so, please, remember that it is not allowed.\n"); + } } // Have the root rank communicate the nc filename diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index efb2a4fd65bb..01bc46e1e601 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -1,6 +1,7 @@ #ifndef SCREAM_IO_UTILS_HPP #define SCREAM_IO_UTILS_HPP +#include "scream_io_control.hpp" #include "share/util/scream_time_stamp.hpp" #include @@ -59,11 +60,17 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } +// The AD will pass a default constructed control, since it doesn't know the values +// of REST_N/REST_OPTION used in the previous run +// Output streams MUST pass a valid control structure, cause we need to differentiate +// between, e.g., streams with same filename prefix, but different output freq specs std::string find_filename_in_rpointer ( - const std::string& casename, + const std::string& filename_prefix, const bool model_restart, const ekat::Comm& comm, - const util::TimeStamp& run_t0); + const util::TimeStamp& run_t0, + const OutputAvgType avg_type = OutputAvgType::Instant, + const IOControl& control = {}); struct LongNames { diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index b24b473d0b1d..7797a0f76bd9 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -171,7 +171,9 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, if (perform_history_restart) { using namespace scorpio; - auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0); + IOFileSpecs hist_restart_specs; + hist_restart_specs.ftype = FileType::HistoryRestart; + auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0,m_avg_type,m_output_control); scorpio::register_file(rhist_file,scorpio::Read); // From restart file, get the time of last write, as well as the current size of the avg sample @@ -196,22 +198,8 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // We do NOT allow changing output specs across restart. If you do want to change // any of these, you MUST start a new output stream (e.g., setting 'Perform Restart: false') - auto old_freq = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_frequency"); - EKAT_REQUIRE_MSG (old_freq == m_output_control.frequency, - "Error! Cannot change frequency when performing history restart.\n" - " - old freq: " << old_freq << "\n" - " - new freq: " << m_output_control.frequency << "\n"); - auto old_freq_units = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_frequency_units"); - EKAT_REQUIRE_MSG (old_freq_units == m_output_control.frequency_units, - "Error! Cannot change frequency units when performing history restart.\n" - " - old freq units: " << old_freq_units << "\n" - " - new freq units: " << m_output_control.frequency_units << "\n"); - auto old_avg_type = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_type"); - EKAT_REQUIRE_MSG (old_avg_type == e2str(m_avg_type), - "Error! Cannot change avg type when performing history restart.\n" - " - old avg type: " << old_avg_type + "\n" - " - new avg type: " << e2str(m_avg_type) << "\n"); - + // NOTE: we do not check that freq/freq_units/avg_type are not changed: since we used + // that info to find the correct rhist file, we already know that they match! auto old_storage_type = scorpio::get_attribute(rhist_file,"GLOBAL","file_max_storage_type"); EKAT_REQUIRE_MSG (old_storage_type == e2str(m_output_file_specs.storage.type), "Error! Cannot change file storage type when performing history restart.\n" diff --git a/components/eamxx/src/share/io/tests/io_utils.cpp b/components/eamxx/src/share/io/tests/io_utils.cpp index 77779307a720..b8187f31c3f4 100644 --- a/components/eamxx/src/share/io/tests/io_utils.cpp +++ b/components/eamxx/src/share/io/tests/io_utils.cpp @@ -9,6 +9,9 @@ TEST_CASE ("find_filename_in_rpointer") { using namespace scream; + constexpr auto AVG = OutputAvgType::Average; + constexpr auto INST = OutputAvgType::Instant; + ekat::Comm comm(MPI_COMM_WORLD); util::TimeStamp t0({2023,9,7},{12,0,0}); @@ -17,21 +20,33 @@ TEST_CASE ("find_filename_in_rpointer") { // Create a dummy rpointer std::ofstream rpointer ("rpointer.atm"); - rpointer << "foo.r." + t0.to_string() + ".nc\n"; - rpointer << "bar2.rhist." + t0.to_string() + ".nc\n"; - rpointer << "bar.rhist." + t0.to_string() + ".nc\n"; - rpointer.close(); + IOControl foo_c, bar_c, bar2_c; + foo_c.frequency = 3; foo_c.frequency_units = "nsteps"; + bar_c.frequency = 1; bar_c.frequency_units = "ndays"; + bar2_c.frequency = 6; bar2_c.frequency_units = "nhours"; - // Now test find_filename_in_rpointer with different inputs + std::string foo_fname = "foo.r.INSTANT.nsteps_x3." + t0.to_string() + ".nc"; + std::string bar_fname = "bar.rhist.AVERAGE.ndays_x1." + t0.to_string() + ".nc"; + std::string bar2_fname = "bar.rhist.AVERAGE.nhours_x6." + t0.to_string() + ".nc"; - REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0)); // wrong prefix - REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t1)); // wrong timestamp - REQUIRE_THROWS (find_filename_in_rpointer("bar",true, comm,t0)); // bar is not model restart - REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0)); // foo is model restart + rpointer << foo_fname<< "\n"; + rpointer << bar_fname<< "\n"; + rpointer << bar2_fname << "\n"; + rpointer.close(); - REQUIRE (find_filename_in_rpointer("bar", false,comm,t0)==("bar.rhist."+t0.to_string()+".nc")); - REQUIRE (find_filename_in_rpointer("bar2",false,comm,t0)==("bar2.rhist."+t0.to_string()+".nc")); - REQUIRE (find_filename_in_rpointer("foo", true, comm,t0)==("foo.r."+t0.to_string()+".nc")); + // Now test find_filename_in_rpointer with different inputs + REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0,AVG)); // missing control (needed for rhist files) + REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0,AVG,bar_c)); // wrong prefix + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t1,AVG,bar_c)); // wrong timestamp + REQUIRE_THROWS (find_filename_in_rpointer("bar",true, comm,t0,AVG,bar_c)); // bar is not model restart + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t0,INST,bar_c)); // wrong avg type + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t0,INST,bar2_c)); // wrong freq specs + REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0,INST,foo_c)); // foo is model restart + REQUIRE_THROWS (find_filename_in_rpointer("foo",true,comm,t0,AVG)); // model restart MUST be INSTANT + + REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar_c)==bar_fname); + REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar2_c)==bar2_fname); + REQUIRE (find_filename_in_rpointer("foo",true, comm,t0)==foo_fname); } TEST_CASE ("io_control") { From f8705dfa43cd16256b8f1bd51a644b996d71d497 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 9 Sep 2024 11:16:26 -0600 Subject: [PATCH 444/477] EAMxx: improve error message in case rhist file is not found --- components/eamxx/src/share/io/scream_io_utils.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 9ff7a641958f..3d79184bac9d 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -77,7 +77,11 @@ std::string find_filename_in_rpointer ( " output freq : " + std::to_string(control.frequency) + "\n" " output freq units: " + control.frequency_units + "\n" " rpointer content:\n" + content + "\n\n" - " Did you change output specs (avg type, freq, or freq units) across restart? If so, please, remember that it is not allowed.\n"); + " Did you change output specs (avg type, freq, or freq units) across restart? If so, please, remember that it is not allowed.\n" + " It is also possible you are using a rhist file create before commit 6b7d441330d. That commit changed how rhist file names\n" + " are formed. In particular, we no longer use INSTANT.${REST_OPTION}_x${REST_N}, but we use the avg type, and freq/freq_option\n" + " of the output stream (to avoid name clashes if 2 streams only differ for one of those). If you want to use your rhist file,\n" + " please rename it, so that the avg-type, freq, and freq_option reflect those of the output stream.\n"); } } From e6c67b9fd399f9b30b93c164fbd99f9af5c70998 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Mon, 30 Sep 2024 16:48:12 -0400 Subject: [PATCH 445/477] Revert "new" craygnuamdgpu build to libfabric/1.15.2.0 to avoid performance bug. --- cime_config/machines/config_machines.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index e2d4a50cab55..88f217e4e97b 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1140,6 +1140,7 @@ PrgEnv-gnu cpe/24.07 + libfabric/1.15.2.0 craype-accel-amd-gfx90a rocm/6.2.0 libunwind @@ -1196,7 +1197,7 @@ - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.10/install/craygnuamdgppu/cpe-24.07/libfabric-1.15.2.0; else echo "$ADIOS2_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} From a01889ed514de3b97d25b8ee0c5a8391f1d8ef51 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 1 Oct 2024 08:44:27 -0600 Subject: [PATCH 446/477] EAMxx: fix compilation error --- components/eamxx/src/share/field/field_impl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 68f7fe6115a3..bfc0d44e99a2 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -654,9 +654,8 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) using RangePolicy = typename KokkosTypes::RangePolicy; auto policy = RangePolicy(0,x_l.size()); - auto ext = layout.extents(); + auto ext = x_l.extents(); - auto policy = RangePolicy(0,x_l.size()); switch (x_l.rank()) { case 0: { From 51b4bf2034ae3fea6d5da7ece3030bb313024b13 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 1 Oct 2024 09:15:46 -0600 Subject: [PATCH 447/477] EAMxx: fix handling of ranks in filename Do not expose option in YAML files anymore. Instead: - for standalone: always add ranks to filename - for CIME tests: never add ranks to filename --- components/eamxx/src/share/io/scream_io_utils.cpp | 4 ++++ .../eamxx/src/share/io/scream_output_manager.cpp | 13 ++++--------- .../output.yaml | 1 - .../homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml | 1 - .../physics_only/mam/optics_rrtmgp/output.yaml | 1 - .../physics_only/shoc_p3_nudging/CMakeLists.txt | 2 -- .../physics_only/shoc_p3_nudging/output.yaml | 1 - .../shoc_p3_nudging/output_remapped.yaml | 1 - .../tests/single-process/mam/optics/output.yaml | 1 - 9 files changed, 8 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 3d79184bac9d..2d5e0a840d09 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -2,6 +2,7 @@ #include "share/io/scream_scorpio_interface.hpp" #include "share/util/scream_utils.hpp" +#include "share/scream_config.hpp" #include #include @@ -35,6 +36,9 @@ std::string find_filename_in_rpointer ( " - filename prefix: " + filename_prefix + "\n"); pattern_str += e2str(avg_type) + "." + control.frequency_units + "_x" + std::to_string(control.frequency); } + if (is_scream_standalone()) { + pattern_str += ".np" + std::to_string(comm.size()); + } pattern_str += "." + run_t0.to_string() + ".nc"; std::regex pattern (pattern_str); diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 7797a0f76bd9..4f35827ba3f4 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -611,8 +611,10 @@ compute_filename (const IOFileSpecs& file_specs, filename += "." + e2str(m_avg_type); filename += "." + control.frequency_units+ "_x" + std::to_string(control.frequency); - // Optionally, add number of mpi ranks (useful mostly in unit tests, to run multiple MPI configs in parallel) - if (m_params.get("MPI Ranks in Filename")) { + // For standalone EAMxx, we may have 2+ versions of the same test running with two + // different choices of ranks. To avoid name clashing for the output files, + // add the comm size to the output file name. + if (is_scream_standalone()) { filename += ".np" + std::to_string(m_io_comm.size()); } @@ -667,7 +669,6 @@ set_params (const ekat::ParameterList& params, m_filename_prefix = m_params.get("filename_prefix"); // Hard code some parameters in case we access them later - m_params.set("MPI Ranks in Filename",false); m_params.set("Floating Point Precision","real"); } else { auto avg_type = m_params.get("Averaging Type"); @@ -703,12 +704,6 @@ set_params (const ekat::ParameterList& params, "Error! Invalid/unsupported value for 'Floating Point Precision'.\n" " - input value: " + prec + "\n" " - supported values: float, single, double, real\n"); - - // If not set, hard code to false for CIME cases, and true for standalone, - // since standalone may be running multiple versions of the same test at once - if (not m_params.isParameter("MPI Ranks in Filename")) { - m_params.set("MPI Ranks in Filename",is_scream_standalone()); - } } // Output control diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml index f2408ab189a5..5922c718e6e6 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml @@ -120,5 +120,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml index 559f5158bb60..be4bb16b95d6 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml @@ -117,5 +117,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml index 24e1dfbb3e7a..bdda48ebe459 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml @@ -26,5 +26,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt index 973e58b7da53..62c4b6d2266e 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt @@ -19,7 +19,6 @@ CreateUnitTest(create_vert_remap_and_weights "create_vert_remap_and_weights.cpp" # Run a test to setup nudging source data: set (NUM_STEPS 5) set (POSTFIX source_data) -set (ADD_RANKS false) set (ATM_TIME_STEP 300) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_source_data.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_source_data.yaml) @@ -36,7 +35,6 @@ CreateUnitTestFromExec (shoc_p3_source shoc_p3_nudging set (NUM_STEPS 5) set (ATM_TIME_STEP 300) set (POSTFIX nudged) -set (ADD_RANKS true) set (VERT_TYPE TIME_DEPENDENT_3D_PROFILE) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_nudging.yaml) diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml index e740c5cdf1f1..d9b573068ae7 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml @@ -3,7 +3,6 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged Averaging Type: Instant Max Snapshots Per File: 2 -MPI Ranks in Filename: ${ADD_RANKS} Field Names: - p_mid - T_mid diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml index c75a6a862f71..26f53fbe57f5 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml @@ -4,7 +4,6 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged_remapped Averaging Type: Instant Max Snapshots Per File: 2 vertical_remap_file: vertical_remap.nc -MPI Ranks in Filename: ${ADD_RANKS} Field Names: - T_mid - qv diff --git a/components/eamxx/tests/single-process/mam/optics/output.yaml b/components/eamxx/tests/single-process/mam/optics/output.yaml index 8ea3d237933d..d0c27e042843 100644 --- a/components/eamxx/tests/single-process/mam/optics/output.yaml +++ b/components/eamxx/tests/single-process/mam/optics/output.yaml @@ -13,5 +13,4 @@ Fields: output_control: Frequency: 2 frequency_units: nsteps - MPI Ranks in Filename: true ... From 20aab95002238f420c3b98877b55ecae031a7c96 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 09:33:33 -0600 Subject: [PATCH 448/477] Pass data_type to subfield --- components/eamxx/src/share/field/field.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 320e10a60273..5f7fb62132f8 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -92,7 +92,7 @@ subfield (const std::string& sf_name, const ekat::units::Units& sf_units, "Error! Subview dimension index must be either 0 or 1.\n"); // Create identifier for subfield - FieldIdentifier sf_id(sf_name,lt.clone().strip_dim(idim),sf_units,id.get_grid_name()); + FieldIdentifier sf_id(sf_name,lt.clone().strip_dim(idim),sf_units,id.get_grid_name(),id.data_type()); // Create empty subfield, then set header and views // Note: we can access protected members, since it's the same type From 639f70f8fa4388b7afed7915a100d1fa7d78d56f Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 10:07:05 -0600 Subject: [PATCH 449/477] Create sync_subfield tests This test will fail with current master. Following changes allow it to pass --- .../eamxx/src/share/tests/field_tests.cpp | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 1cb69f474dee..094e4ef057cf 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -812,4 +812,79 @@ TEST_CASE ("update") { } } + +TEST_CASE ("sync_subfields") { + // This test is for previously incorrect behavior, where syncing a subfield + // to host/device would deep copy the entire data view (including all entries of + // the parent view). Here, if memory space is not shared between host and device, + // syncing a subfield to host/device will not sync the data of the other subfields. + + using namespace scream; + using namespace ekat::units; + using namespace ShortFieldTagsNames; + using FID = FieldIdentifier; + using FL = FieldLayout; + + constexpr int ncols = 10; + constexpr int ndims = 4; + constexpr int nlevs = 8; + + // Create field with (col, cmp, lev) + FID fid ("V",FL({COL,CMP,LEV},{ncols,ndims,nlevs}),Units::nondimensional(),"the_grid",DataType::IntType); + Field f (fid); + f.allocate_view(); + + // Store whether mem space for host and device are the same for testing subfield values + const bool shared_mem_space = + f.get_view().data() == f.get_view().data(); + + // Deep copy all values to ndims on device and host + f.deep_copy(ndims); + f.sync_to_host(); + + // Set subfield values to their index on device + for (int c=0; c(); + for (int idx=0; idx(c); + } + + // Sync only component 0 to device + f.get_component(0).sync_to_dev(); + + // For components 1,...,ndims-1, if device and host do not share a + // memory space, device values should be equal to ndims, else device + // values should be equal to component index + for (int c=1; c(); + Kokkos::parallel_for(Kokkos::MDRangePolicy>({0,0}, {ncols,nlevs}), + KOKKOS_LAMBDA (const int icol, const int ilev) { + if (shared_mem_space) EKAT_KERNEL_ASSERT(device_subview(icol, ilev) == c); + else EKAT_KERNEL_ASSERT(device_subview(icol, ilev) == ndims); + }); + } +} + } // anonymous namespace From 4cb9fb85241350a23f692e29c41cfa6ff8b59d3e Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 10:38:44 -0600 Subject: [PATCH 450/477] New sync_to_host impl Now, syncing to host for subfield only deep copies subfield values (not entire data array for parent) --- components/eamxx/src/share/field/field.cpp | 17 +++- components/eamxx/src/share/field/field.hpp | 3 + .../eamxx/src/share/field/field_impl.hpp | 96 +++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 5f7fb62132f8..5a489ec618ae 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -65,7 +65,22 @@ sync_to_host () const { EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); - Kokkos::deep_copy(m_data.h_view,m_data.d_view); + // Check for early return if Host and Device are the same memory space + if (m_data.h_view.data() == m_data.d_view.data()) return; + + switch (data_type()) { + case DataType::IntType: + sync_to_host_impl(); + break; + case DataType::FloatType: + sync_to_host_impl(); + break; + case DataType::DoubleType: + sync_to_host_impl(); + break; + default: + EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); + } } void Field:: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index e48a570d8131..be17cdddbfb7 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -282,6 +282,9 @@ class Field { // Cuda requires methods enclosing __device__ lambda's to be public protected: #endif + template + void sync_to_host_impl () const; + template void deep_copy_impl (const ST value); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 67753605864c..b6f4f14a0068 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -222,6 +222,102 @@ get_strided_view_type { return DstView(get_ND_view()); } +template +void Field::sync_to_host_impl () const { + const auto alloc_props = get_header().get_alloc_properties(); + switch (rank()) { + case 0: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 1: + if (alloc_props.contiguous()) { + Kokkos::deep_copy(get_view(), get_view()); + } else { + auto host_view = get_strided_view< ST*, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST**, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST***, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST****, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST*****, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto host_view = get_strided_view< ST******, Host>(); + auto dev_view = get_strided_view(); + for (size_t i=0; i void Field:: deep_copy (const Field& src) { From 15883006dc879aec0ec03019bb6241dc6c2946ce Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 10:45:58 -0600 Subject: [PATCH 451/477] New sync_to_dev impl Now, syncing to device for subfield only deep copies subfield values (not entire data array for parent) --- components/eamxx/src/share/field/field.cpp | 18 +++- components/eamxx/src/share/field/field.hpp | 3 + .../eamxx/src/share/field/field_impl.hpp | 96 +++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 5a489ec618ae..81f994cc3a50 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -89,8 +89,22 @@ sync_to_dev () const { EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); - // Ensure host view was created (lazy construction) - Kokkos::deep_copy(m_data.d_view,m_data.h_view); + // Check for early return if Host and Device are the same memory space + if (m_data.h_view.data() == m_data.d_view.data()) return; + + switch (data_type()) { + case DataType::IntType: + sync_to_dev_impl(); + break; + case DataType::FloatType: + sync_to_dev_impl(); + break; + case DataType::DoubleType: + sync_to_dev_impl(); + break; + default: + EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_dev.\n"); + } } Field Field:: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index be17cdddbfb7..1cbf36236f36 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -285,6 +285,9 @@ class Field { template void sync_to_host_impl () const; + template + void sync_to_dev_impl () const; + template void deep_copy_impl (const ST value); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index b6f4f14a0068..8dbfc3e35a6d 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -318,6 +318,102 @@ void Field::sync_to_host_impl () const { } } +template +void Field::sync_to_dev_impl () const { + const auto alloc_props = get_header().get_alloc_properties(); + switch (rank()) { + case 0: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 1: + if (alloc_props.contiguous()) { + Kokkos::deep_copy(get_view(), get_view()); + } else { + auto dev_view = get_strided_view< ST*, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST**, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST***, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST****, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST*****, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + } else { + auto dev_view = get_strided_view< ST******, Device>(); + auto host_view = get_strided_view(); + for (size_t i=0; i void Field:: deep_copy (const Field& src) { From cfa4b46c2048415890a660619a3337b97a4947b9 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 1 Oct 2024 16:25:22 -0600 Subject: [PATCH 452/477] Allow sync_to_host for read-only fields Use mutable keyword for m_is_read_only to temporarily disable for sync_to_host when device memspace != host memspace --- components/eamxx/src/share/field/field.cpp | 7 +++++++ components/eamxx/src/share/field/field.hpp | 2 +- .../eamxx/src/share/grid/remap/coarsening_remapper.cpp | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 81f994cc3a50..40551eed039b 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -68,6 +68,10 @@ sync_to_host () const { // Check for early return if Host and Device are the same memory space if (m_data.h_view.data() == m_data.d_view.data()) return; + // We allow sync_to_host for constant fields. Temporarily disable read only flag. + const bool original_read_only = m_is_read_only; + m_is_read_only = false; + switch (data_type()) { case DataType::IntType: sync_to_host_impl(); @@ -81,6 +85,9 @@ sync_to_host () const { default: EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); } + + // Return field to read-only state + m_is_read_only = original_read_only; } void Field:: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 1cbf36236f36..e40e9649fede 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -344,7 +344,7 @@ class Field { dual_view_t m_data; // Whether this field is read-only - bool m_is_read_only = false; + mutable bool m_is_read_only = false; }; // We use this to find a Field in a std container. diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index f915cb14d292..7d6590c327a6 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -3,6 +3,7 @@ #include "share/grid/point_grid.hpp" #include "share/grid/grid_import_export.hpp" #include "share/io/scorpio_input.hpp" +#include "share/field/field.hpp" #include #include From 1230ce601b42e344bbf60a902341ea79e56bd39c Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Wed, 2 Oct 2024 16:07:16 -0600 Subject: [PATCH 453/477] Add comment on need for mutable m_is_read_only --- components/eamxx/src/share/field/field.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index e40e9649fede..833e49a611bb 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -343,7 +343,9 @@ class Field { // Actual data. dual_view_t m_data; - // Whether this field is read-only + // Whether this field is read-only. This is given + // mutable keyword since it needs to be turned off/on + // to allow sync_to_host for constant, read-only fields. mutable bool m_is_read_only = false; }; From c97aa4c0687d8d341add03840bc1db78096c2bcd Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Wed, 2 Oct 2024 16:21:03 -0600 Subject: [PATCH 454/477] Consolidate sync_to_host_impl and sync_to_device_impl into sync_views_impl --- components/eamxx/src/share/field/field.cpp | 12 +- components/eamxx/src/share/field/field.hpp | 7 +- .../eamxx/src/share/field/field_impl.hpp | 198 +++++------------- 3 files changed, 59 insertions(+), 158 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 40551eed039b..1b3de68ba08f 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -74,13 +74,13 @@ sync_to_host () const { switch (data_type()) { case DataType::IntType: - sync_to_host_impl(); + sync_views_impl(); break; case DataType::FloatType: - sync_to_host_impl(); + sync_views_impl(); break; case DataType::DoubleType: - sync_to_host_impl(); + sync_views_impl(); break; default: EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); @@ -101,13 +101,13 @@ sync_to_dev () const { switch (data_type()) { case DataType::IntType: - sync_to_dev_impl(); + sync_views_impl(); break; case DataType::FloatType: - sync_to_dev_impl(); + sync_views_impl(); break; case DataType::DoubleType: - sync_to_dev_impl(); + sync_views_impl(); break; default: EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_dev.\n"); diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 833e49a611bb..90e5ccd652c0 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -282,11 +282,8 @@ class Field { // Cuda requires methods enclosing __device__ lambda's to be public protected: #endif - template - void sync_to_host_impl () const; - - template - void sync_to_dev_impl () const; + template + void sync_views_impl () const; template void deep_copy_impl (const ST value); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 8dbfc3e35a6d..becb7b9f2f8b 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -222,94 +222,94 @@ get_strided_view_type { return DstView(get_ND_view()); } -template -void Field::sync_to_host_impl () const { +template +void Field::sync_views_impl () const { const auto alloc_props = get_header().get_alloc_properties(); switch (rank()) { case 0: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); break; case 1: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST*, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST**, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST***, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST****, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST*****, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i(), get_view()); + Kokkos::deep_copy(get_view(), get_view()); } else { - auto host_view = get_strided_view< ST******, Host>(); - auto dev_view = get_strided_view(); - for (size_t i=0; i(); + auto from_view = get_strided_view(); + for (size_t i=0; i -void Field::sync_to_dev_impl () const { - const auto alloc_props = get_header().get_alloc_properties(); - switch (rank()) { - case 0: - Kokkos::deep_copy(get_view(), get_view()); - break; - case 1: - if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); - } else { - auto dev_view = get_strided_view< ST*, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST**, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST***, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST****, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST*****, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto dev_view = get_strided_view< ST******, Device>(); - auto host_view = get_strided_view(); - for (size_t i=0; i void Field:: deep_copy (const Field& src) { From 756ea4accfdd9582d708ffec869104363642f850 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 2 Oct 2024 17:02:35 -0600 Subject: [PATCH 455/477] EAMxx: fix usage of layout extents in field methods --- .../eamxx/src/share/field/field_impl.hpp | 25 ++++++++++++++++--- .../eamxx/src/share/field/field_layout.cpp | 6 ++--- .../eamxx/src/share/field/field_layout.hpp | 2 ++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index bfc0d44e99a2..5eb810be9d84 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -306,10 +306,19 @@ deep_copy_impl (const Field& src) { auto src_alloc_props = src.get_header().get_alloc_properties(); auto tgt_alloc_props = get_header().get_alloc_properties(); - using RangePolicy = typename KokkosTypes::RangePolicy; + using device_t = typename Field::get_device; + using exec_space = typename device_t::execution_space; + using RangePolicy = Kokkos::RangePolicy; + auto policy = RangePolicy(0,layout.size()); - auto ext = layout.extents(); + using extents_type = typename ekat::KokkosTypes::template view_1d; + extents_type ext; + if constexpr (HD==Device) { + ext = layout.extents(); + } else { + ext = layout.extents_h(); + } switch (rank) { case 1: { @@ -651,10 +660,18 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) " - x layout: " + x_l.to_string() + "\n" " - y layout: " + y_l.to_string() + "\n"); - using RangePolicy = typename KokkosTypes::RangePolicy; + using device_t = typename Field::get_device; + using exec_space = typename device_t::execution_space; + using RangePolicy = Kokkos::RangePolicy; auto policy = RangePolicy(0,x_l.size()); - auto ext = x_l.extents(); + using extents_type = typename ekat::KokkosTypes::template view_1d; + extents_type ext; + if constexpr (HD==Device) { + ext = x_l.extents(); + } else { + ext = x_l.extents_h(); + } switch (x_l.rank()) { case 0: diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 2ce924cd0a74..7cd9e953f657 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -250,9 +250,9 @@ FieldLayout& FieldLayout::reset_dim (const FieldTag t, const int extent, const b void FieldLayout::set_extents () { m_extents = decltype(m_extents)("",m_rank); - auto extents_h = Kokkos::create_mirror_view(m_extents); - std::copy_n(m_dims.begin(),m_rank,extents_h.data()); - Kokkos::deep_copy(m_extents,extents_h); + m_extents_h = Kokkos::create_mirror_view(m_extents); + std::copy_n(m_dims.begin(),m_rank,m_extents_h.data()); + Kokkos::deep_copy(m_extents,m_extents_h); } void FieldLayout::compute_type () { diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index 018831f8818c..481ca0c922e8 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -97,6 +97,7 @@ class FieldLayout { int dim (const int idim) const; const std::vector& dims () const { return m_dims; } const extents_type& extents () const { return m_extents; } + const extents_type::HostMirror& extents_h () const { return m_extents_h; } long long size () const; @@ -156,6 +157,7 @@ class FieldLayout { std::vector m_names; std::vector m_dims; extents_type m_extents; + extents_type::HostMirror m_extents_h; LayoutType m_type; }; From c99106082c6b5efcfddb7cf8631bd854b1fd95fa Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 3 Oct 2024 08:49:30 -0600 Subject: [PATCH 456/477] EAMxx: fix nc file names in some standalone tests --- .../eamxx/src/physics/nudging/tests/nudging_tests.cpp | 2 +- components/eamxx/src/share/io/tests/io_utils.cpp | 8 +++++--- .../physics_only/shoc_p3_nudging/input_nudging.yaml | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index 51773cb249df..a5a158f44a17 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -51,7 +51,7 @@ TEST_CASE("nudging_tests") { const int nlevs_fine = 2*nlevs_data -1; // Files names - auto postfix = ".INSTANT.nsteps_x1." + get_t0().to_string() + ".nc"; + auto postfix = ".INSTANT.nsteps_x1.np*." + get_t0().to_string() + ".nc"; auto nudging_data = "nudging_data" + postfix; auto nudging_data_filled = "nudging_data_filled" + postfix; auto map_file = "map_ncol" + std::to_string(ngcols_data) diff --git a/components/eamxx/src/share/io/tests/io_utils.cpp b/components/eamxx/src/share/io/tests/io_utils.cpp index b8187f31c3f4..cf7b66ad8596 100644 --- a/components/eamxx/src/share/io/tests/io_utils.cpp +++ b/components/eamxx/src/share/io/tests/io_utils.cpp @@ -25,9 +25,10 @@ TEST_CASE ("find_filename_in_rpointer") { bar_c.frequency = 1; bar_c.frequency_units = "ndays"; bar2_c.frequency = 6; bar2_c.frequency_units = "nhours"; - std::string foo_fname = "foo.r.INSTANT.nsteps_x3." + t0.to_string() + ".nc"; - std::string bar_fname = "bar.rhist.AVERAGE.ndays_x1." + t0.to_string() + ".nc"; - std::string bar2_fname = "bar.rhist.AVERAGE.nhours_x6." + t0.to_string() + ".nc"; + std::string suffix = ".np" + std::to_string(comm.size()) + "." + t0.to_string() + ".nc"; + std::string foo_fname = "foo.r.INSTANT.nsteps_x3" + suffix; + std::string bar_fname = "bar.rhist.AVERAGE.ndays_x1" + suffix; + std::string bar2_fname = "bar.rhist.AVERAGE.nhours_x6" + suffix; rpointer << foo_fname<< "\n"; rpointer << bar_fname<< "\n"; @@ -44,6 +45,7 @@ TEST_CASE ("find_filename_in_rpointer") { REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0,INST,foo_c)); // foo is model restart REQUIRE_THROWS (find_filename_in_rpointer("foo",true,comm,t0,AVG)); // model restart MUST be INSTANT + auto test = find_filename_in_rpointer("bar",false,comm,t0,AVG,bar_c); REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar_c)==bar_fname); REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar2_c)==bar2_fname); REQUIRE (find_filename_in_rpointer("foo",true, comm,t0)==foo_fname); diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml index 2b33d15f9f75..6a4c0003376e 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml @@ -14,7 +14,7 @@ atmosphere_processes: p3: max_total_ni: 720.0e3 nudging: - nudging_filenames_patterns: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.${RUN_T0}.nc] + nudging_filenames_patterns: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.np1.${RUN_T0}.nc] nudging_fields: ["T_mid", "qv"] nudging_timescale: 1000 use_nudging_weights: true From 48930bfd6ecb5cb8f28631339555439a95fbbc7f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 3 Oct 2024 09:53:59 -0600 Subject: [PATCH 457/477] EAMxx: fix usage of regex when parsing rpointer file --- components/eamxx/src/share/io/scream_io_utils.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 2d5e0a840d09..9318728d657a 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -51,7 +51,7 @@ std::string find_filename_in_rpointer ( while (std::getline(rpointer_file,line)) { content += line + "\n"; - if (std::regex_match(line,pattern)) { + if (std::regex_search(line,pattern)) { filename = line; found = true; break; @@ -70,12 +70,14 @@ std::string find_filename_in_rpointer ( EKAT_ERROR_MSG ( "Error! Restart requested, but no model restart file found in 'rpointer.atm'.\n" " model restart filename prefix: " + filename_prefix + "\n" + " model restart filename pattern: " + pattern_str + "\n" " run t0 : " + run_t0.to_string() + "\n" " rpointer content:\n" + content + "\n\n"); } else { EKAT_ERROR_MSG ( "Error! Restart requested, but no history restart file found in 'rpointer.atm'.\n" " hist restart filename prefix: " + filename_prefix + "\n" + " hist restart filename pattern: " + pattern_str + "\n" " run t0 : " + run_t0.to_string() + "\n" " avg_type : " + e2str(avg_type) + "\n" " output freq : " + std::to_string(control.frequency) + "\n" From f9a57c730917c3cb253f10b6cdf603077b6d88df Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 10:23:48 -0600 Subject: [PATCH 458/477] Try async deep copy for sync to host --- .../eamxx/src/share/field/field_impl.hpp | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index becb7b9f2f8b..cafce7d4622b 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -224,14 +224,16 @@ get_strided_view_type { template void Field::sync_views_impl () const { + using ExecSpace = typename Field::get_device::execution_space; + const auto alloc_props = get_header().get_alloc_properties(); switch (rank()) { case 0: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); break; case 1: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST*, To >(); auto from_view = get_strided_view(); @@ -239,13 +241,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i); auto from_view_i = Kokkos::subview(from_view, i); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 2: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST**, To >(); auto from_view = get_strided_view(); @@ -253,13 +255,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 3: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST***, To >(); auto from_view = get_strided_view(); @@ -267,13 +269,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL(), Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL(), Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 4: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST****, To >(); auto from_view = get_strided_view(); @@ -281,13 +283,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 5: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST*****, To >(); auto from_view = get_strided_view(); @@ -295,13 +297,13 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; case 6: if (alloc_props.contiguous()) { - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); } else { auto to_view = get_strided_view< ST******, To >(); auto from_view = get_strided_view(); @@ -309,13 +311,17 @@ void Field::sync_views_impl () const { auto to_view_i = Kokkos::subview(to_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); auto from_view_i = Kokkos::subview(from_view, i, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); EKAT_REQUIRE_MSG(to_view_i.span_is_contiguous(), "Error! EAMxx field must be contiguous after the first dimension.\n"); - Kokkos::deep_copy(to_view_i, from_view_i); + Kokkos::deep_copy(ExecSpace(), to_view_i, from_view_i); } } break; default: EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); } + // Since deep_copy are all async wrt host, we need to fence before returning + if constexpr (To == Host) { + Kokkos::fence(); + } } template From 3b66686189b23afb56d4f203acdbccc450d995e2 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 3 Oct 2024 13:07:22 -0600 Subject: [PATCH 459/477] Remove most allocations from rrtmgp interface --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/scream_rrtmgp_interface.hpp | 178 ++++++++++++------ 2 files changed, 123 insertions(+), 57 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 8ff525eeed1d..8d88d427e78f 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 8ff525eeed1d87a2ca6f251c4d16b46222c5554d +Subproject commit 8d88d427e78fe7958564e587d9f776c0b36d3762 diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 216722b3766b..a420c0c1dd4f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -565,6 +565,44 @@ static void rrtmgp_sw( int ngpt = k_dist.get_ngpt(); int ngas = gas_concs.get_num_gases(); + // Allocate temporaries from pool + const int size1 = nday; + const int size2 = nday*nlay; // 4 + const int size3 = nday*(nlay+1); // 5 + const int size4 = ncol*nlay; + const int size5 = nbnd*nday; //2 + const int size6 = nday*ngpt; + const int size7 = nday*(nlay+1)*nbnd; // 3 + const int size8 = ncol*nlay*(k_dist.get_ngas()+1); + + RealT* data = pool_t::template alloc_raw(size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8), *dcurr = data; + + auto mu0_day = view_t (dcurr, nday); dcurr += size1; + + auto p_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto vmr_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_limited = view_t (dcurr, nday, nlay); dcurr += size2; + + auto p_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto t_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_up_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_dir_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + + auto vmr = view_t (dcurr, ncol, nlay); dcurr += size4; + + auto sfc_alb_dir_T = view_t (dcurr, nbnd, nday); dcurr += size5; + auto sfc_alb_dif_T = view_t (dcurr, nbnd, nday); dcurr += size5; + + auto toa_flux = view_t (dcurr, nday, ngpt); dcurr += size6; + + auto bnd_flux_up_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_dir_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + + auto col_gas = view_t(dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size8; + // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; auto &flux_dn = fluxes.flux_dn; @@ -623,20 +661,15 @@ static void rrtmgp_sw( } // Subset mu0 - auto mu0_day = view_t("mu0_day", nday); Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { mu0_day(iday) = mu0(dayIndices(iday)); }); // subset state variables - auto p_lay_day = view_t("p_lay_day", nday, nlay); - auto t_lay_day = view_t("t_lay_day", nday, nlay); Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); }); - auto p_lev_day = view_t("p_lev_day", nday, nlay+1); - auto t_lev_day = view_t("t_lev_day", nday, nlay+1); Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); @@ -647,8 +680,6 @@ static void rrtmgp_sw( gas_concs_t gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = view_t("vmr_day", nday, nlay); - auto vmr = view_t("vmr" , ncol, nlay); gas_concs.get_vmr(gas_names[igas], vmr); Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); @@ -680,20 +711,12 @@ static void rrtmgp_sw( // RRTMGP assumes surface albedos have a screwy dimension ordering // for some strange reason, so we need to transpose these; also do // daytime subsetting in the same kernel - view_t sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - view_t sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); }); // Temporaries we need for daytime-only fluxes - auto flux_up_day = view_t("flux_up_day", nday, nlay+1); - auto flux_dn_day = view_t("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = view_t("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = view_t("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = view_t("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = view_t("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); fluxes_t fluxes_day; fluxes_day.flux_up = flux_up_day; fluxes_day.flux_dn = flux_dn_day; @@ -713,18 +736,14 @@ static void rrtmgp_sw( } // Limit temperatures for gas optics look-up tables - auto t_lay_limited = view_t("t_lay_limited", nday, nlay); limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); // Do gas optics - view_t toa_flux("toa_flux", nday, ngpt); bool top_at_1 = false; Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { val |= p_lay(0, 0) < p_lay(0, nlay-1); }, Kokkos::LOr(top_at_1)); - oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); if (extra_clnsky_diag) { k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); @@ -803,6 +822,8 @@ static void rrtmgp_sw( clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); }); } + + pool_t::dealloc(data, dcurr - data); } /* @@ -819,6 +840,24 @@ static void rrtmgp_lw( { // Problem size int nbnd = k_dist.get_nband(); + int constexpr max_gauss_pts = 4; + + const int size1 = ncol; + const int size2 = nbnd*ncol; + const int size3 = max_gauss_pts*max_gauss_pts; + const int size4 = ncol*nlay; + const int size5 = ncol*(nlay+1); + const int size6 = ncol*nlay*(k_dist.get_ngas()+1); + + RealT* data = pool_t::template alloc_raw(size1 + size2 + size3*2 + size4 + size5 + size6), *dcurr = data; + + view_t t_sfc (dcurr, ncol); dcurr += size1; + view_t emis_sfc (dcurr, nbnd,ncol); dcurr += size2; + view_t gauss_Ds (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t gauss_wts (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t t_lay_limited(dcurr, ncol, nlay); dcurr += size4; + view_t t_lev_limited(dcurr, ncol, nlay+1); dcurr += size5; + view_t col_gas (dcurr, std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); dcurr += size6; // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; @@ -863,8 +902,6 @@ static void rrtmgp_lw( // Boundary conditions source_func_t lw_sources; lw_sources.alloc(ncol, nlay, k_dist); - view_t t_sfc ("t_sfc" ,ncol); - view_t emis_sfc("emis_sfc",nbnd,ncol); bool top_at_1 = false; Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { @@ -882,32 +919,31 @@ static void rrtmgp_lw( // Weights and angle secants for first order (k=1) Gaussian quadrature. // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - hview_t gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - hview_t gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - view_t gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - view_t gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + RealT gauss_Ds_host_raw[max_gauss_pts][max_gauss_pts] = { + {1.66, 1.18350343, 1.09719858, 1.06056257}, + {0., 2.81649655, 1.69338507, 1.38282560}, + {0., 0., 4.70941630, 2.40148179}, + {0., 0., 0., 7.15513024} + }; + hview_t gauss_Ds_host (&gauss_Ds_host_raw[0][0], max_gauss_pts, max_gauss_pts); + + RealT gauss_wts_host_raw[max_gauss_pts][max_gauss_pts] = { + {0.5, 0.3180413817, 0.2009319137, 0.1355069134}, + {0., 0.1819586183, 0.2292411064, 0.2034645680}, + {0., 0., 0.0698269799, 0.1298475476}, + {0., 0., 0., 0.0311809710} + }; + + hview_t gauss_wts_host(&gauss_wts_host_raw[0][0],max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); Kokkos::deep_copy(gauss_wts, gauss_wts_host); // Limit temperatures for gas optics look-up tables - auto t_lay_limited = view_t("t_lay_limited", ncol, nlay); - auto t_lev_limited = view_t("t_lev_limited", ncol, nlay+1); limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); // Do gas optics - oview_t col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); if (extra_clnsky_diag) { k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); @@ -941,23 +977,22 @@ static void rrtmgp_lw( // Compute clean-sky fluxes rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); } + + pool_t::dealloc(data, dcurr - data); } /* * Return a subcolumn mask consistent with a specified overlap assumption */ -static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds) +static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds, int3dk& subcolumn_mask) { - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - int3dk subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - // Subcolumn generators are a means for producing a variable x(i,j,k), where // // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) // // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = view_t("cldx", ncol, nlay, ngpt); + auto cldx = pool_t::template alloc(ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1017,6 +1052,9 @@ static int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, subcolumn_mask(icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldx); + return subcolumn_mask; } @@ -1029,7 +1067,7 @@ static void compute_cloud_area( { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = view_t("subcol_mask", ncol, ngpt); + auto subcol_mask = pool_t::template alloc(ncol, ngpt); Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but // using play/pmid does not @@ -1046,6 +1084,8 @@ static void compute_cloud_area( cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; } }); + + pool_t::dealloc(subcol_mask); } /* @@ -1078,7 +1118,7 @@ static void compute_aerocom_cloudtop( Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = view_t("aerocom_clr", ncol); + auto aerocom_clr = pool_t::template alloc(ncol); Kokkos::deep_copy(aerocom_clr, 1.0); // Get gravity acceleration constant from constants @@ -1151,6 +1191,8 @@ static void compute_aerocom_cloudtop( // (their products) cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); + + pool_t::dealloc(aerocom_clr); } /* @@ -1253,14 +1295,17 @@ static optical_props2_t get_cloud_optics_sw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = view_t("rel_limited", ncol, nlay); - auto rei_limited = view_t("rei_limited", ncol, nlay); + auto rel_limited = pool_t::template alloc(ncol, nlay); + auto rei_limited = pool_t::template alloc(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + // Return optics return clouds; } @@ -1279,14 +1324,17 @@ static optical_props1_t get_cloud_optics_lw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = view_t("rel_limited", ncol, nlay); - auto rei_limited = view_t("rei_limited", ncol, nlay); + auto rel_limited = pool_t::alloc(ncol, nlay); + auto rei_limited = pool_t::alloc(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + // Return optics return clouds; } @@ -1298,6 +1346,10 @@ static optical_props2_t get_subsampled_clouds( optical_props2_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_2str(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::alloc(ncol, nlay, ngpt); + // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud @@ -1305,7 +1357,7 @@ static optical_props2_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + auto cldfrac_rad = pool_t::alloc(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1319,11 +1371,11 @@ static optical_props2_t get_subsampled_clouds( int overlap = 1; // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = view_t("seeds", ncol); + auto seeds = pool_t::alloc(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { @@ -1338,6 +1390,11 @@ static optical_props2_t get_subsampled_clouds( subsampled_optics.g (icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + return subsampled_optics; } @@ -1349,6 +1406,10 @@ static optical_props1_t get_subsampled_clouds( optical_props1_t subsampled_optics; subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); subsampled_optics.alloc_1scl(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::alloc(ncol, nlay, ngpt); + // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud @@ -1356,7 +1417,7 @@ static optical_props1_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = view_t("cldfrac_rad", ncol, nlay); + auto cldfrac_rad = pool_t::alloc(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1367,11 +1428,11 @@ static optical_props1_t get_subsampled_clouds( // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM; use different // seed values for longwave and shortwave - auto seeds = view_t("seeds", ncol); + auto seeds = pool_t::alloc(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { @@ -1382,6 +1443,11 @@ static optical_props1_t get_subsampled_clouds( subsampled_optics.tau(icol,ilay,igpt) = 0; } }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + return subsampled_optics; } From 3cf1406f74f2875b316cf3d06bc2622b196800a9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 3 Oct 2024 15:11:25 -0600 Subject: [PATCH 460/477] Update rrtmgp to latest kokkos-coversion-branch --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 8d88d427e78f..e64b99cce24e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 8d88d427e78fe7958564e587d9f776c0b36d3762 +Subproject commit e64b99cce24eb31bb6f317bddb6f0ffbdfaf8bb7 From 463ffbc702fefb9c42926315148859169fdc1d12 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Thu, 3 Oct 2024 17:01:07 -0700 Subject: [PATCH 461/477] A quick change to recreate an aborted run mid simulation --- components/eamxx/cime_config/namelist_defaults_scream.xml | 1 + components/eamxx/src/control/atmosphere_driver.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 57611fc3aa91..9a13029cec6f 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -604,6 +604,7 @@ be lost if SCREAM_HACK_XML is not enabled. + -9999 0 0, "Error! Input time step must be positive.\n"); From 9daab1a446be1621bdedfe879f21e7f899a15ec3 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:33:39 -0600 Subject: [PATCH 462/477] Add function to return if host == device mem space --- components/eamxx/src/share/field/field.cpp | 4 ++-- components/eamxx/src/share/field/field.hpp | 7 +++++++ components/eamxx/src/share/tests/field_tests.cpp | 3 +-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 1b3de68ba08f..a002c3b28813 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -66,7 +66,7 @@ sync_to_host () const { "Error! Input field must be allocated in order to sync host and device views.\n"); // Check for early return if Host and Device are the same memory space - if (m_data.h_view.data() == m_data.d_view.data()) return; + if (host_and_device_share_memory_space()) return; // We allow sync_to_host for constant fields. Temporarily disable read only flag. const bool original_read_only = m_is_read_only; @@ -97,7 +97,7 @@ sync_to_dev () const { "Error! Input field must be allocated in order to sync host and device views.\n"); // Check for early return if Host and Device are the same memory space - if (m_data.h_view.data() == m_data.d_view.data()) return; + if (host_and_device_share_memory_space()) return; switch (data_type()) { case DataType::IntType: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 90e5ccd652c0..679f293e3e1c 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -278,6 +278,13 @@ class Field { // Allocate the actual view void allocate_view (); + inline bool host_and_device_share_memory_space() const { + EKAT_REQUIRE_MSG(is_allocated(), + "Error! Must allocate view before querying " + "host_and_device_share_memory_space().\n"); + return m_data.h_view.data() == m_data.d_view.data(); + } + #ifndef KOKKOS_ENABLE_CUDA // Cuda requires methods enclosing __device__ lambda's to be public protected: diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 094e4ef057cf..0dfe2522de39 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -835,8 +835,7 @@ TEST_CASE ("sync_subfields") { f.allocate_view(); // Store whether mem space for host and device are the same for testing subfield values - const bool shared_mem_space = - f.get_view().data() == f.get_view().data(); + const bool shared_mem_space = f.host_and_device_share_memory_space(); // Deep copy all values to ndims on device and host f.deep_copy(ndims); From c265fb58c10490766e26ec3100a0dbfb41a3319b Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:36:47 -0600 Subject: [PATCH 463/477] Create contiguous helper field --- components/eamxx/src/share/field/field.cpp | 16 ++++++++++++++ components/eamxx/src/share/field/field.hpp | 25 ++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index a002c3b28813..bf1ecbb5dfaa 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -137,6 +137,14 @@ subfield (const std::string& sf_name, const ekat::units::Units& sf_units, sf.m_data = m_data; sf.m_is_read_only = m_is_read_only; + if (not sf.m_header->get_alloc_properties().contiguous() and + not sf.host_and_device_share_memory_space()) { + // If subfield is not contiguous and Host and Device do not + // share a memory space, we must initialize the helper field + // for sync_to functions. + sf.initialize_contiguous_helper_field(); + } + return sf; } @@ -178,6 +186,14 @@ Field Field::subfield(const std::string& sf_name, index_end); sf.m_data = m_data; + if (not sf.m_header->get_alloc_properties().contiguous() and + not sf.host_and_device_share_memory_space()) { + // If subfield is not contiguous and Host and Device do not + // share a memory space, we must initialize the helper field + // for sync_to functions. + sf.initialize_contiguous_helper_field(); + } + return sf; } diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 679f293e3e1c..a3779f1ebf24 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -278,6 +278,27 @@ class Field { // Allocate the actual view void allocate_view (); + // Create contiguous helper field for running sync_to_host + // and sync_to_device with non-contiguous fields + void initialize_contiguous_helper_field () { + EKAT_REQUIRE_MSG(not m_header->get_alloc_properties().contiguous(), + "Error! We should not setup contiguous helper field " + "for an already contiguous field.\n"); + EKAT_REQUIRE_MSG(not host_and_device_share_memory_space(), + "Error! We should not setup contiguous helper field for a field " + "when host and device share a memory space.\n"); + + auto id = m_header->get_identifier(); + Field contig(id.alias(name()+std::string("_contiguous"))); + contig.allocate_view(); + + // Sanity check + EKAT_REQUIRE_MSG(contig.get_header().get_alloc_properties().contiguous(), + "Error! Contiguous helper field must be contiguous.\n"); + + m_contiguous_field = std::make_shared(contig); + } + inline bool host_and_device_share_memory_space() const { EKAT_REQUIRE_MSG(is_allocated(), "Error! Must allocate view before querying " @@ -347,6 +368,10 @@ class Field { // Actual data. dual_view_t m_data; + // Field needed for sync host/device in case of non-contiguous + // field when host and device do not share a memory space. + std::shared_ptr m_contiguous_field; + // Whether this field is read-only. This is given // mutable keyword since it needs to be turned off/on // to allow sync_to_host for constant, read-only fields. From e5e8a10fe0c24ba68dfb4d64cbde4f3574da00da Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:39:09 -0600 Subject: [PATCH 464/477] Make deep_copy const --- components/eamxx/src/share/field/field.hpp | 8 ++++---- components/eamxx/src/share/field/field_impl.hpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index a3779f1ebf24..c071a79c840c 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -198,11 +198,11 @@ class Field { // Set the field to a constant value (on host or device) template - void deep_copy (const T value); + void deep_copy (const T value) const; // Copy the data from one field to this field template - void deep_copy (const Field& src); + void deep_copy (const Field& src) const; // Updates this field y as y=alpha*x+beta*y // NOTE: ST=void is just so we can give a default to HD, @@ -314,10 +314,10 @@ class Field { void sync_views_impl () const; template - void deep_copy_impl (const ST value); + void deep_copy_impl (const ST value) const; template - void deep_copy_impl (const Field& src); + void deep_copy_impl (const Field& src) const; template void update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index cafce7d4622b..d5917b944177 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -326,7 +326,7 @@ void Field::sync_views_impl () const { template void Field:: -deep_copy (const Field& src) { +deep_copy (const Field& src) const { EKAT_REQUIRE_MSG (not m_is_read_only, "Error! Cannot call deep_copy on read-only fields.\n"); @@ -350,7 +350,7 @@ deep_copy (const Field& src) { template void Field:: -deep_copy (const ST value) { +deep_copy (const ST value) const { EKAT_REQUIRE_MSG (not m_is_read_only, "Error! Cannot call deep_copy on read-only fields.\n"); @@ -384,7 +384,7 @@ deep_copy (const ST value) { template void Field:: -deep_copy_impl (const Field& src) { +deep_copy_impl (const Field& src) const { const auto& layout = get_header().get_identifier().get_layout(); const auto& layout_src = src.get_header().get_identifier().get_layout(); @@ -528,7 +528,7 @@ deep_copy_impl (const Field& src) { } template -void Field::deep_copy_impl (const ST value) { +void Field::deep_copy_impl (const ST value) const { // Note: we can't just do a deep copy on get_view_impl(), since this // field might be a subfield of another. Instead, get the From a1ca668f646c05e01ca329f0f7213402ca067c02 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:39:45 -0600 Subject: [PATCH 465/477] Pass data_type to other subfield impl --- components/eamxx/src/share/field/field.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index bf1ecbb5dfaa..686e70bd5881 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -177,7 +177,7 @@ Field Field::subfield(const std::string& sf_name, auto sf_layout = lt.clone(); sf_layout.reset_dim(idim, index_end - index_beg); // Create identifier for subfield - FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name()); + FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name(), id.data_type()); // Create empty subfield, then set header and views // Note: we can access protected members, since it's the same type From 9333fa54fbe75decee8ae7a861a1022df6cd5865 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 3 Oct 2024 16:56:30 -0600 Subject: [PATCH 466/477] Change sync_views_impl to use scratch contiguous field --- .../eamxx/src/share/field/field_impl.hpp | 139 ++++++------------ 1 file changed, 42 insertions(+), 97 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index d5917b944177..076ffc9d2da0 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -224,103 +224,48 @@ get_strided_view_type { template void Field::sync_views_impl () const { - using ExecSpace = typename Field::get_device::execution_space; - - const auto alloc_props = get_header().get_alloc_properties(); - switch (rank()) { - case 0: - Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); - break; - case 1: - if (alloc_props.contiguous()) { - Kokkos::deep_copy(ExecSpace(), get_view(), get_view()); - } else { - auto to_view = get_strided_view< ST*, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST**, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST***, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST****, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST*****, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); - } else { - auto to_view = get_strided_view< ST******, To >(); - auto from_view = get_strided_view(); - for (size_t i=0; i(), get_view()); + break; + case 1: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 2: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 3: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 4: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 5: + Kokkos::deep_copy(get_view(), get_view()); + break; + case 6: + Kokkos::deep_copy(get_view(), get_view()); + break; + default: + EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); + } + } else { + // Store values in contiguous helper field + m_contiguous_field->deep_copy(*this); + + // Sync helper field + if constexpr (To==Host) { + m_contiguous_field->sync_to_host(); + } else { + m_contiguous_field->sync_to_dev(); + } + + // Copy values back to this field + deep_copy(*m_contiguous_field); } } From e52c5cf6bb0c4fee3d7fc73dae254b299f1a2244 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 3 Oct 2024 14:26:28 -0600 Subject: [PATCH 467/477] EAMxx: allow non-contiguous set of cpus in test-all-scream --- components/eamxx/scripts/test_all_scream.py | 28 ++++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index f5350954b1c3..61ce833e5640 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -398,7 +398,7 @@ def generate_cmake_config(self, test, for_ctest=False): return result ############################################################################### - def get_taskset_range(self, test, for_compile=True): + def get_taskset_resources(self, test, for_compile=True): ############################################################################### res_name = "compile_res_count" if for_compile else "testing_res_count" @@ -419,13 +419,11 @@ def get_taskset_range(self, test, for_compile=True): expect(offset < len(affinity_cp), f"Offset {offset} out of bounds (max={len(affinity_cp)}) for test {test}\naffinity_cp: {affinity_cp}") - start = affinity_cp[offset] - end = start - for i in range(1, getattr(test, res_name)): - expect(affinity_cp[offset+i] == start+i, f"Could not get contiguous range for test {test}") - end = affinity_cp[offset+i] + resources = [] + for i in range(0, getattr(test, res_name)): + resources.append(affinity_cp[offset+i]) - return start, end + return resources ############################################################################### def create_ctest_resource_file(self, test, build_dir): @@ -438,7 +436,7 @@ def create_ctest_resource_file(self, test, build_dir): # res group is where we usually bind an individual MPI rank. # The id of the res groups on is offset so that it is unique across all builds - start, end = self.get_taskset_range(test, for_compile=False) + resources = self.get_taskset_resources(test, for_compile=False) data = {} @@ -448,7 +446,7 @@ def create_ctest_resource_file(self, test, build_dir): # We add leading zeroes to ensure that ids will sort correctly # both alphabetically and numerically devices = [] - for res_id in range(start,end+1): + for res_id in resources: devices.append({"id":f"{res_id:05d}"}) # Add resource groups @@ -457,7 +455,7 @@ def create_ctest_resource_file(self, test, build_dir): with (build_dir/"ctest_resource_file.json").open("w", encoding="utf-8") as outfile: json.dump(data,outfile,indent=2) - return (end-start)+1 + return len(resources) ############################################################################### def generate_ctest_config(self, cmake_config, extra_configs, test): @@ -498,11 +496,11 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): # Ctest can only competently manage test pinning across a single instance of ctest. For # multiple concurrent instances of ctest, we have to help it. It's OK to use the compile_res_count - # taskset range even though the ctest script is also running the tests + # taskset resources even though the ctest script is also running the tests if self._parallel: - start, end = self.get_taskset_range(test) + resources = self.get_taskset_resources(test) result = result.replace("'", r"'\''") # handle nested quoting - result = f"taskset -c {start}-{end} sh -c '{result}'" + result = f"taskset -c {','.join([str(r) for r in resources])} sh -c '{result}'" return result @@ -539,8 +537,8 @@ def generate_baselines(self, test): cmd = f"make -j{test.compile_res_count}" if self._parallel: - start, end = self.get_taskset_range(test) - cmd = f"taskset -c {start}-{end} sh -c '{cmd}'" + resources = self.get_taskset_resources(test) + cmd = f"taskset -c {','.join([str(r) for r in resources])} sh -c '{cmd}'" stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True) From f1b3a4bb720ec7f8f579072b2f291e436da36999 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Fri, 4 Oct 2024 13:53:21 -0700 Subject: [PATCH 468/477] Change the name of fail option to be self-descriptive --- .../cime_config/namelist_defaults_scream.xml | 15 +++++++++++++-- .../eamxx/src/control/atmosphere_driver.cpp | 9 ++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 9a13029cec6f..7e92482e6767 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -48,7 +48,7 @@ be lost if SCREAM_HACK_XML is not enabled. ctl_nl - driver_options,iop_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters + driver_debug_options,driver_options,iop_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters @@ -602,9 +602,20 @@ be lost if SCREAM_HACK_XML is not enabled. + + + + + -9999 + + + - -9999 0 ("force_crash_nsteps", -9999)==m_current_ts.get_num_steps(); + if (force_fail_nstep) { + abort(); + } } // Make sure the end of the time step is after the current start_time EKAT_REQUIRE_MSG (dt>0, "Error! Input time step must be positive.\n"); From 61874d7d569ac84b694dc74913894e10e8a923d1 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 4 Oct 2024 12:41:14 -0600 Subject: [PATCH 469/477] Use asynchronous deep_copy in non-contig sync_views_impl --- components/eamxx/src/share/field/field.cpp | 8 +- components/eamxx/src/share/field/field.hpp | 7 +- .../eamxx/src/share/field/field_impl.hpp | 97 +++++++++++++++---- 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 686e70bd5881..00a2c9b593fc 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -60,7 +60,7 @@ Field::clone(const std::string& name) const { } void Field:: -sync_to_host () const { +sync_to_host (const bool fence) const { // Sanity check EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); @@ -86,12 +86,14 @@ sync_to_host () const { EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); } + if (fence) Kokkos::fence(); + // Return field to read-only state m_is_read_only = original_read_only; } void Field:: -sync_to_dev () const { +sync_to_dev (const bool fence) const { // Sanity check EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); @@ -112,6 +114,8 @@ sync_to_dev () const { default: EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_dev.\n"); } + + if (fence) Kokkos::fence(); } Field Field:: diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index c071a79c840c..a338199f2ef0 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -193,8 +193,11 @@ class Field { // Note: this class takes no responsibility in keeping track of whether // a sync is required in either direction. Mainly because we expect // host views to be seldom used, and even less frequently modified. - void sync_to_host () const; - void sync_to_dev () const; + // The fence input controls whether a fence is done at the end of the sync. + // If multiple syncs are performed in a row on different data, the user may + // want to run them asynchronously and fence the final sync_to call. + void sync_to_host (const bool fence = true) const; + void sync_to_dev (const bool fence = true) const; // Set the field to a constant value (on host or device) template diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 076ffc9d2da0..ba484466747e 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -224,48 +224,103 @@ get_strided_view_type { template void Field::sync_views_impl () const { + // All deep_copy call we want to be async for host. + using DeviceExecSpace = typename Field::get_device::execution_space; + + // Rank 0 will always be contiguous. Copy and return early. + if (rank() == 0) { + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + return; + } const bool is_contiguous = get_header().get_alloc_properties().contiguous(); if (is_contiguous) { - // For contiguous fields, simply use Kokkos::deep_copy() + // For contiguous fields, simply use Kokkos::deep_copy(). switch (rank()) { - case 0: - Kokkos::deep_copy(get_view(), get_view()); - break; case 1: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 2: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 3: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 4: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 5: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; case 6: - Kokkos::deep_copy(get_view(), get_view()); + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); break; default: EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); } } else { - // Store values in contiguous helper field - m_contiguous_field->deep_copy(*this); - - // Sync helper field - if constexpr (To==Host) { - m_contiguous_field->sync_to_host(); - } else { - m_contiguous_field->sync_to_dev(); + auto sync_helper = [this] () { + if constexpr (To==Host) m_contiguous_field->sync_to_host(); + else m_contiguous_field->sync_to_dev(); + }; + switch (rank()) { + case 1: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 2: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 3: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 4: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 5: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 6: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + default: + EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); } - - // Copy values back to this field - deep_copy(*m_contiguous_field); } } From 1db9a5aae70cd5c89018eccf1a61c68bc5019723 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 4 Oct 2024 15:35:11 -0600 Subject: [PATCH 470/477] better comment --- components/eamxx/src/share/field/field_impl.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index ba484466747e..1c0d8257647f 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -224,7 +224,8 @@ get_strided_view_type { template void Field::sync_views_impl () const { - // All deep_copy call we want to be async for host. + // For all Kokkos::deep_copy() calls we will pass in an instance of the + // device execution space so that we are asynchronous w.r.t. host. using DeviceExecSpace = typename Field::get_device::execution_space; // Rank 0 will always be contiguous. Copy and return early. From f3ef2e69ab453fe034fbf9554a4f05a070965e33 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Mon, 7 Oct 2024 11:34:22 -0700 Subject: [PATCH 471/477] Simply call to new abort feature in the AD code --- components/eamxx/src/control/atmosphere_driver.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index faf2a725ef6c..a4bd4fabc528 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -1595,12 +1595,12 @@ void AtmosphereDriver::run (const int dt) { start_timer("EAMxx::run"); // DEBUG option: Check if user has set the run to fail at a specific timestep. - if (m_atm_params.isSublist("driver_debug_options")) { - bool force_fail_nstep = m_atm_params.sublist("driver_debug_options").get("force_crash_nsteps", -9999)==m_current_ts.get_num_steps(); - if (force_fail_nstep) { - abort(); - } + auto& debug = m_atm_params.sublist("driver_debug_options"); + auto fail_step = debug.get("force_crash_nsteps",-1); + if (fail_step==m_current_ts.get_num_steps()) { + std::abort(); } + // Make sure the end of the time step is after the current start_time EKAT_REQUIRE_MSG (dt>0, "Error! Input time step must be positive.\n"); From 6f3edeae3bd2a56ff3fb803bbba3d5986a8090d4 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 9 Oct 2024 17:09:34 -0400 Subject: [PATCH 472/477] keep github actions working for e3sm repo --- .github/workflows/e3sm-gh-ci-cime-tests.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 04f7fcb4ffcf..6c9ee1ab114c 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -25,7 +25,7 @@ on: jobs: ci: - if: false + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index d39717589914..852bceec5359 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -13,7 +13,7 @@ on: jobs: scream-defaults: - if: false + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} From 64f041b9f8db1089902c956fe293d900d78ae159 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 10 Oct 2024 11:22:10 -0700 Subject: [PATCH 473/477] Make sure SCREAM_LIBS_ONLY is on We don't want testing stuff or small kernel stuff --- components/eam/src/physics/crm/pam/CMakeLists.txt | 3 ++- components/eamxx/CMakeLists.txt | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 4a8f88f669b0..ac629fa89fde 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -9,8 +9,9 @@ set(PAM_DRIVER_SRC params.F90) add_library(pam_driver - ${PAM_DRIVER_SRC}) + ${PAM_DRIVER_SRC}) +set(SCREAM_LIBS_ONLY TRUE) set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 07c3cda76728..9b6f54e5b056 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -200,6 +200,9 @@ option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them") set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") +message("JGF DEFAUL_SMALL_KERNELS is ${DEFAULT_SMALL_KERNELS}") +message("JGF SCREAM_SMALL_KERNELS is ${SCREAM_SMALL_KERNELS}") +message("JGF SCREAM_P3_SMALL_KERNELS is ${SCREAM_P3_SMALL_KERNELS}") set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") From 10099ac1b8d7a9a8b5ba3b49ac01e38eb30dba8d Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 10 Oct 2024 11:22:10 -0700 Subject: [PATCH 474/477] Make sure SCREAM_LIBS_ONLY is on We don't want testing stuff or small kernel stuff --- components/eam/src/physics/crm/pam/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 4a8f88f669b0..c55594523843 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -11,6 +11,7 @@ set(PAM_DRIVER_SRC add_library(pam_driver ${PAM_DRIVER_SRC}) +set(SCREAM_LIBS_ONLY TRUE) set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) From c34cb650b81d231ccc8bc49e93c317be2b13c97e Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Thu, 10 Oct 2024 15:43:33 -0700 Subject: [PATCH 475/477] bug fix for pam_statistics.h --- .../eam/src/physics/crm/pam/pam_statistics.h | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/components/eam/src/physics/crm/pam/pam_statistics.h b/components/eam/src/physics/crm/pam/pam_statistics.h index bb277600dfa1..8166b9912d26 100644 --- a/components/eam/src/physics/crm/pam/pam_statistics.h +++ b/components/eam/src/physics/crm/pam/pam_statistics.h @@ -458,29 +458,34 @@ inline void pam_statistics_compute_means( pam::PamCoupler &coupler ) { if (clear_rh_cnt(k,iens)>0) { clear_rh(k,iens) = clear_rh(k,iens) / clear_rh_cnt(k,iens); } - phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); - - phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); - - phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); - - phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + if (phys_tend_sgs_cnt(iens)>0) { + phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); + } + if (phys_tend_micro_cnt(iens)>0) { + phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); + } + if (phys_tend_dycor_cnt(iens)>0) { + phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); + } + if (phys_tend_sponge_cnt(iens)>0) { + phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + } }); //------------------------------------------------------------------------------------------------ } From 12c6dfd3f28ade7398e4c443ffa11d7cf2796f1a Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Fri, 11 Oct 2024 08:08:33 -0700 Subject: [PATCH 476/477] update PAM submodule --- components/eam/src/physics/crm/pam/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 3ea20ad38f28..c3b6522c57b7 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 3ea20ad38f286730973429e0d491420a6f599f11 +Subproject commit c3b6522c57b754d073e1deaad5ce8125f7f88325 From 1b5fb62f8e37402a1a1f2cdf17405ce851112385 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 11 Oct 2024 11:51:37 -0400 Subject: [PATCH 477/477] enable shared builds for gh-standalone --- .github/workflows/eamxx-gh-ci-standalone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml index 9fde89ed35bf..d276d600357e 100644 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -44,7 +44,7 @@ jobs: run: | # TODO: get rid of this extra line if we can? git config --global safe.directory '*' - ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} + ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} -c BUILD_SHARED_LIBS=ON - name: Artifacts uses: actions/upload-artifact@v4